diff --git a/.github/assemble-docs.sh b/.github/assemble-docs.sh index 63adc76f1..18917c350 100755 --- a/.github/assemble-docs.sh +++ b/.github/assemble-docs.sh @@ -35,28 +35,7 @@ cp android/README.md site_docs/android/README.md cp bindings/flutter/README.md site_docs/flutter/README.md cp bindings/swift/README.md site_docs/swift/README.md cp bindings/kotlin/README.md site_docs/kotlin/README.md - -mkdir -p site_docs/react-native -if curl -sfL "https://raw.githubusercontent.com/cactus-compute/cactus-react-native/main/README.md" -o site_docs/react-native/README.md; then - - { - echo '!!! info "Independent release cycle"' - echo ' The React Native SDK releases independently from the Cactus engine.' - echo ' Check the [releases page](https://github.com/cactus-compute/cactus-react-native/releases) for the latest compatible version.' - echo '' - cat site_docs/react-native/README.md - } > site_docs/react-native/README.tmp && mv site_docs/react-native/README.tmp site_docs/react-native/README.md - - mkdir -p site_docs/react-native/assets - curl -sfL "https://raw.githubusercontent.com/cactus-compute/cactus-react-native/main/assets/logo.png" -o site_docs/react-native/assets/logo.png 2>/dev/null || true - echo "Fetched React Native README" -else - echo "# React Native SDK" > site_docs/react-native/README.md - echo "" >> site_docs/react-native/README.md - echo "See [cactus-react-native on GitHub](https://github.com/cactus-compute/cactus-react-native) for full documentation." >> site_docs/react-native/README.md - echo "Warning: Could not fetch React Native README, using fallback" -fi - +cp bindings/react-native/README.md site_docs/react-native/README.md cp bindings/rust/README.md site_docs/rust/README.md if [ -d blog ] && ls blog/*.md >/dev/null 2>&1; then @@ -93,8 +72,6 @@ sedi 's|(/bindings/rust/)|(rust/README.md)|g' site_docs/index.md sedi 's|(/python/)|(python/README.md)|g' site_docs/index.md sedi 's|(/apple/)|(apple/README.md)|g' site_docs/index.md sedi 's|(/android/)|(android/README.md)|g' site_docs/index.md -sedi 's|(/flutter/)|(flutter/README.md)|g' site_docs/index.md -sedi 's|(/rust/)|(rust/README.md)|g' site_docs/index.md sedi 's|(/blog/hybrid_transcription\.md)|(blog/hybrid_transcription.md)|g' site_docs/index.md sedi 's|(/blog/lfm2_24b_a2b\.md)|(blog/lfm2_24b_a2b.md)|g' site_docs/index.md sedi 's|(/blog/parakeet\.md)|(blog/parakeet.md)|g' site_docs/index.md @@ -127,8 +104,6 @@ for f in site_docs/docs/*.md; do sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" - sedi 's|(/flutter/)|(../flutter/README.md)|g' "$f" - sedi 's|(/rust/)|(../rust/README.md)|g' "$f" done for f in site_docs/python/README.md site_docs/apple/README.md site_docs/android/README.md site_docs/flutter/README.md site_docs/swift/README.md site_docs/kotlin/README.md; do @@ -152,8 +127,6 @@ for f in site_docs/python/README.md site_docs/apple/README.md site_docs/android/ sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" - sedi 's|(/flutter/)|(../flutter/README.md)|g' "$f" - sedi 's|(/rust/)|(../rust/README.md)|g' "$f" sedi 's|(\.\.\/README\.md)|(../index.md)|g' "$f" done @@ -179,8 +152,6 @@ if [ -f site_docs/rust/README.md ]; then 's|(/python/)|(../python/README.md)|g' \ 's|(/apple/)|(../apple/README.md)|g' \ 's|(/android/)|(../android/README.md)|g' \ - 's|(/flutter/)|(../flutter/README.md)|g' \ - 's|(/rust/)|(../rust/README.md)|g' \ 's|(\.\.\/README\.md)|(../index.md)|g'; do sedi "$pattern" site_docs/rust/README.md done @@ -206,8 +177,6 @@ if ls site_docs/blog/*.md >/dev/null 2>&1; then sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" - sedi 's|(/flutter/)|(../flutter/README.md)|g' "$f" - sedi 's|(/rust/)|(../rust/README.md)|g' "$f" sedi 's|(/blog/hybrid_transcription\.md)|(hybrid_transcription.md)|g' "$f" sedi 's|(/blog/lfm2_24b_a2b\.md)|(lfm2_24b_a2b.md)|g' "$f" sedi 's|(/blog/parakeet\.md)|(parakeet.md)|g' "$f" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5ff4c3958..e05a224e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,6 @@ name: Build on: pull_request: paths: - - cactus/** - cactus-engine/** - cactus-graph/** - cactus-kernels/** @@ -13,7 +12,6 @@ on: push: branches: [main] paths: - - cactus/** - cactus-engine/** - cactus-graph/** - cactus-kernels/** @@ -25,13 +23,17 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: apple: runs-on: macos-15 - timeout-minutes: 15 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - run: bash cactus/build.sh + - run: bash cactus-engine/build.sh - run: BUILD_STATIC=true BUILD_XCFRAMEWORK=false bash apple/build.sh - run: bash android/build.sh @@ -41,4 +43,4 @@ jobs: steps: - uses: actions/checkout@v4 - run: sudo apt-get update && sudo apt-get install -y cmake build-essential libcurl4-openssl-dev - - run: bash cactus/build.sh + - run: bash cactus-engine/build.sh diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 636bb764b..a75ad5312 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -1,23 +1,29 @@ +name: Unit Tests + on: pull_request: paths: - - cactus/** - - tests/** + - cactus-kernels/** + - cactus-graph/** - .github/workflows/cpp.yml push: branches: [main] paths: - - cactus/** - - tests/** + - cactus-kernels/** + - cactus-graph/** - .github/workflows/cpp.yml workflow_dispatch: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - test: - timeout-minutes: 45 + unit-tests: + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -30,17 +36,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: ${{ matrix.deps }} - - uses: actions/cache@v4 - with: - path: ~/.ccache - key: ccache-${{ matrix.os }}-${{ github.ref_name }}-${{ hashFiles('cactus/CMakeLists.txt', 'tests/CMakeLists.txt', '.github/workflows/cpp.yml') }} - restore-keys: | - ccache-${{ matrix.os }}-${{ github.ref_name }}- - ccache-${{ matrix.os }}- - - run: ccache --zero-stats || true - - run: cmake -G Ninja -S cactus -B cactus/build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - run: cmake --build cactus/build --target cactus_ffi --parallel 4 - - run: cmake -G Ninja -S tests -B tests/build-ci -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - run: cmake --build tests/build-ci --target test_stt --parallel 4 - - run: ./tests/build-ci/test_stt - - run: ccache --show-stats || true + - name: Run kernel tests + run: bash cactus-kernels/test.sh + - name: Run graph tests + run: bash cactus-graph/test.sh diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index ed73abd3e..fdbf6d45c 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -4,9 +4,17 @@ on: pull_request: types: [opened, synchronize, reopened] +permissions: + contents: read + +concurrency: + group: dco-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: dco-check: runs-on: ubuntu-latest + timeout-minutes: 5 name: DCO Check steps: - name: Checkout diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 67fb7296f..b084bbd51 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,8 +13,13 @@ on: - python/README.md - apple/README.md - android/README.md + - bindings/README.md - bindings/flutter/README.md + - bindings/kotlin/README.md + - bindings/python/README.md + - bindings/react-native/README.md - bindings/rust/README.md + - bindings/swift/README.md - README.md workflow_call: inputs: @@ -23,9 +28,17 @@ on: required: false workflow_dispatch: +permissions: + contents: read + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: false + jobs: deploy: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: write steps: @@ -36,7 +49,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.x" + python-version: "3.12" - name: Install dependencies run: pip install "mkdocs==1.6.1" "mkdocs-material==9.7.5" "mike==2.1.4" diff --git a/.github/workflows/inference.yml b/.github/workflows/inference.yml index 47c49cb40..10241bbfa 100644 --- a/.github/workflows/inference.yml +++ b/.github/workflows/inference.yml @@ -8,9 +8,14 @@ on: permissions: contents: read +concurrency: + group: inference-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: gate: runs-on: ubuntu-latest + timeout-minutes: 5 outputs: approved: ${{ steps.check.outputs.approved }} steps: @@ -28,6 +33,7 @@ jobs: needs: gate if: needs.gate.outputs.approved == 'true' runs-on: [self-hosted, linux, arm64] + timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: @@ -39,77 +45,35 @@ jobs: source ./setup fi - - name: Convert models - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - source venv/bin/activate - convert() { - local id=$1 dir=$2 bits=$3 - [ -f "weights/$dir/config.txt" ] && return - python3 -m cactus convert "$id" --bits "$bits" ${HF_TOKEN:+--token "$HF_TOKEN"} || true - } - convert "Qwen/Qwen3-0.6B" "qwen3-0.6b" "4" - convert "google/gemma-3-270m-it" "gemma-3-270m-it" "4" - convert "openai/whisper-small" "whisper-small" "4" - convert "nvidia/parakeet-tdt-0.6b-v3" "parakeet-tdt-0.6b-v3" "4" - - - name: Build + - name: Build libcactus + engine tests run: | - bash cactus/build.sh + bash cactus-engine/build.sh cd cactus-engine/tests rm -rf build && mkdir -p build && cd build cmake .. -DCMAKE_BUILD_TYPE=Release make -j$(nproc) - unit-tests: - needs: build - runs-on: [self-hosted, linux, arm64] - env: - CACTUS_INDEX_PATH: /tmp/cactus_test_index - CACTUS_NO_CLOUD_TELE: "1" - steps: - - uses: actions/checkout@v4 - with: - clean: false - - name: Run unit tests - run: | - mkdir -p "$CACTUS_INDEX_PATH" - cd cactus-engine/tests/build - ./test_index - ./test_model_loading - ./test_telemetry - - - name: Run kernel tests - run: bash cactus-kernels/test.sh - - - name: Run graph tests - run: bash cactus-graph/test.sh - - inference-tests: + model-tests: needs: build runs-on: [self-hosted, linux, arm64] + timeout-minutes: 60 env: - CACTUS_TEST_ASSETS: ${{ github.workspace }}/cactus-engine/tests/assets CACTUS_NO_CLOUD_TELE: "1" strategy: fail-fast: false matrix: - include: - - test: test_llm - model_dir: qwen3-0.6b - - test: test_llm - model_dir: gemma-3-270m-it - - test: test_stt - model_dir: whisper-small - - test: test_stt - model_dir: parakeet-tdt-0.6b-v3 + model: + - LiquidAI/LFM2-VL-450M + - openai/whisper-small steps: - uses: actions/checkout@v4 with: clean: false - - name: Run ${{ matrix.test }} (${{ matrix.model_dir }}) + + - name: Run tests for ${{ matrix.model }} env: - CACTUS_TEST_MODEL: ${{ github.workspace }}/weights/${{ matrix.model_dir }} - CACTUS_TEST_TRANSCRIBE_MODEL: ${{ github.workspace }}/weights/${{ matrix.model_dir }} - run: ./cactus-engine/tests/build/${{ matrix.test }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + source venv/bin/activate + MODEL="${{ github.event.inputs.model || matrix.model }}" + python3 -m cactus test --component engine --model "$MODEL" diff --git a/.github/workflows/publish_to_hf.yml b/.github/workflows/publish_to_hf.yml deleted file mode 100644 index 5100a4535..000000000 --- a/.github/workflows/publish_to_hf.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: Publish to Hugging Face - -on: - release: - types: [published] - workflow_dispatch: - inputs: - model: - description: Model to publish (leave empty to publish all) - required: false - type: string - version: - description: Version tag - required: true - type: string - org: - description: Organization - required: false - default: Cactus-Compute - type: string - int4: - description: Export INT4 - required: false - type: boolean - default: false - int8: - description: Export INT8 - required: false - type: boolean - default: false - fp16: - description: Export FP16 - required: false - type: boolean - default: false - apple: - description: Export Apple weights - required: false - type: boolean - default: false - pipeline_tag: - description: Pipeline tag (e.g. text-generation) - required: false - type: string - tags: - description: Comma-separated tags (e.g. completion,tools) - required: false - type: string - description: - description: Model description - required: false - type: string - branch: - description: Branch override (uses version tag if empty) - required: false - type: string - -jobs: - publish: - runs-on: macos-latest - steps: - - name: Get version - id: version - run: | - if [ "${{ github.event_name }}" == "release" ]; then - echo "tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT - else - echo "tag=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT - fi - - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.branch || steps.version.outputs.tag }} - - - uses: actions/checkout@v4 - with: - repository: cactus-compute/cactus-pro - token: ${{ secrets.CACTUS_PRO_TOKEN }} - path: cactus-pro - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: pip install -r python/requirements.txt - - - name: Publish models - run: | - cd python - ORG="${{ github.event.inputs.org || 'Cactus-Compute' }}" - VERSION="${{ steps.version.outputs.tag }}" - - publish_model() { - local model=$1 int4=$2 int8=$3 fp16=$4 apple=$5 pipeline_tag=$6 tags=$7 description="${8:-}" - local FLAGS=(--version "$VERSION" --org "$ORG" --model "$model") - [ "$int4" == "true" ] && FLAGS+=(--int4) - [ "$int8" == "true" ] && FLAGS+=(--int8) - [ "$fp16" == "true" ] && FLAGS+=(--fp16) - [ "$apple" == "true" ] && FLAGS+=(--apple) - [ -n "$pipeline_tag" ] && FLAGS+=(--pipeline-tag "$pipeline_tag") - [ -n "$tags" ] && FLAGS+=(--tags "$tags") - set +e - python -m src.publish_to_hf --task export_model "${FLAGS[@]}" --description "$description" - local rc=$? - set -e - return $rc - } - - if [ -n "${{ github.event.inputs.model }}" ]; then - publish_model \ - "${{ github.event.inputs.model }}" \ - "${{ github.event.inputs.int4 }}" \ - "${{ github.event.inputs.int8 }}" \ - "${{ github.event.inputs.fp16 }}" \ - "${{ github.event.inputs.apple }}" \ - "${{ github.event.inputs.pipeline_tag }}" \ - "${{ github.event.inputs.tags }}" \ - "${{ github.event.inputs.description }}" - else - FAILED_MODELS=() - while read -r config; do - model=$(echo "$config" | jq -r '.model') - int4=$(echo "$config" | jq -r '.int4') - int8=$(echo "$config" | jq -r '.int8') - fp16=$(echo "$config" | jq -r '.fp16') - apple=$(echo "$config" | jq -r '.apple') - pipeline_tag=$(echo "$config" | jq -r '.pipeline_tag // ""') - tags=$(echo "$config" | jq -r 'if .tags then .tags | join(",") else "" end') - description=$(echo "$config" | jq -r '.description // ""') - if ! publish_model "$model" "$int4" "$int8" "$fp16" "$apple" "$pipeline_tag" "$tags" "$description"; then - echo "::error::Failed to publish $model" - FAILED_MODELS+=("$model") - fi - done < <(jq -c '.[]' "$GITHUB_WORKSPACE/models.json") - - if [ ${#FAILED_MODELS[@]} -gt 0 ]; then - echo "::error::Failed models: ${FAILED_MODELS[*]}" - exit 1 - fi - fi - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_HUB_DISABLE_XET: "1" - - - name: Update organization README - if: "!github.event.inputs.model" - run: | - cd python - ORG="${{ github.event.inputs.org || 'Cactus-Compute' }}" - python -m src.publish_to_hf --task update_org_readme --org "$ORG" - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index e7c3c0c51..5b62c06ee 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -9,9 +9,13 @@ on: required: false type: string +permissions: + contents: read + jobs: wheels-macos: runs-on: macos-26 + timeout-minutes: 45 env: MACOSX_DEPLOYMENT_TARGET: "14.0" steps: @@ -28,7 +32,7 @@ jobs: pip install -e python/ cactus build mkdir -p python/cactus/bindings/lib - cp cactus/build/libcactus.dylib python/cactus/bindings/lib/ + cp cactus-engine/build/libcactus_engine.dylib python/cactus/bindings/lib/ - name: Build and repair wheel working-directory: python @@ -44,6 +48,7 @@ jobs: wheels-linux: runs-on: ubuntu-24.04-arm + timeout-minutes: 45 container: image: quay.io/pypa/manylinux_2_28_aarch64 steps: @@ -61,7 +66,7 @@ jobs: pip install -e python/ cactus build mkdir -p python/cactus/bindings/lib - cp cactus/build/libcactus.so python/cactus/bindings/lib/ + cp cactus-engine/build/libcactus_engine.so python/cactus/bindings/lib/ - name: Build and repair wheel working-directory: python @@ -77,6 +82,7 @@ jobs: sdist: runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - uses: actions/checkout@v4 @@ -100,6 +106,7 @@ jobs: publish: needs: [wheels-macos, wheels-linux, sdist] runs-on: ubuntu-24.04 + timeout-minutes: 15 permissions: id-token: write steps: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 5e0672ca4..ab063e29e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -10,10 +10,19 @@ on: paths: - "python/**" - ".github/workflows/python.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: test: runs-on: ubuntu-24.04-arm + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -21,7 +30,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y cmake build-essential libcurl4-openssl-dev - name: Build libcactus - run: bash cactus/build.sh + run: bash cactus-engine/build.sh - uses: actions/setup-python@v5 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 217ccac37..325e2db38 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,9 @@ on: release: types: [published] +permissions: + contents: read + concurrency: group: release-${{ github.ref }} cancel-in-progress: false @@ -18,6 +21,7 @@ jobs: homebrew: needs: pypi runs-on: ubuntu-24.04 + timeout-minutes: 15 env: PACKAGE: cactus-compute TAP: cactus-compute/homebrew-cactus @@ -65,6 +69,8 @@ jobs: license "Cactus Compute License" include Language::Python::Virtualenv depends_on "python@3.12" + depends_on macos: :ventura + depends_on arch: :arm64 livecheck do url "https://pypi.org/pypi/${PACKAGE}/json" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index bb39ea933..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,45 +0,0 @@ -on: - workflow_dispatch: - push: - branches: [main] - paths: [rust/**, cactus/**, .github/workflows/rust.yml] - pull_request: - paths: [rust/**, cactus/**, .github/workflows/rust.yml] - -permissions: - contents: read - -jobs: - test: - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - os: macos-15 - deps: brew install llvm - - os: ubuntu-24.04-arm - deps: sudo apt-get update && sudo apt-get install -y cmake build-essential libcurl4-openssl-dev libclang-dev - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: ${{ matrix.deps }} - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - - run: pip3 install --break-system-packages huggingface-hub && pip3 install --break-system-packages -e python/ --no-deps - - uses: actions/cache@v4 - with: - path: weights/ - key: cactus-models-${{ matrix.os }}-v2 - - run: cactus download google/gemma-3-270m-it - - run: cactus download openai/whisper-small - - run: curl -fL -o "${{ runner.temp }}/jfk.wav" https://github.com/ggml-org/whisper.cpp/raw/refs/tags/v1.8.3/samples/jfk.wav - - run: cargo test --manifest-path rust/Cargo.toml --test llm -- --nocapture - env: - CACTUS_MODEL_PATH: ${{ github.workspace }}/weights/gemma-3-270m-it - - run: cargo test --manifest-path rust/Cargo.toml --test stt -- --nocapture - env: - CACTUS_STT_MODEL_PATH: ${{ github.workspace }}/weights/whisper-small - CACTUS_STT_AUDIO_PATH: ${{ runner.temp }}/jfk.wav diff --git a/.github/workflows/verify-pr.yml b/.github/workflows/verify-pr.yml deleted file mode 100644 index 015bc07cd..000000000 --- a/.github/workflows/verify-pr.yml +++ /dev/null @@ -1,225 +0,0 @@ -name: PR Verification -# SETUP -# 1. System packages: -# sudo apt-get install -y build-essential libcurl4-openssl-dev cmake libicu-dev -# 2. Python 3.12 via Homebrew (deadsnakes PPA doesn't support Ubuntu 25.10): -# # Install Homebrew -# NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - -# # Install Python 3.12 -# eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" -# brew install python@3.12 - -# # Make it available system-wide -# sudo ln -sf /home/linuxbrew/.linuxbrew/bin/python3.12 /usr/local/bin/python3.12 - -# GitHub Actions Runner Installation (already done) - -# mkdir -p ~/actions-runner && cd ~/actions-runner -# curl -sO -L https://github.com/actions/runner/releases/download/v2.331.0/actions-runner-linux-arm64-2.331.0.tar.gz -# tar xzf actions-runner-linux-arm64-2.331.0.tar.gz -# sudo ./bin/installdependencies.sh - -# Configure Runner for a Repo (repeat this for each repo) - -# 1. Remove old runner config (if switching repos): -# cd ~/actions-runner -# sudo ./svc.sh stop -# sudo ./svc.sh uninstall -# ./config.sh remove --token -# 1. Get the remove token from the old repo: Settings → Actions → Runners → click runner → Remove. -# 2. Get new token: Go to the target repo (e.g. cactus-compute/cactus) → Settings → Actions → Runners → New self-hosted runner → Linux / ARM64. Copy the token from the ./config.sh command -# shown. -# 3. Configure: -# cd ~/actions-runner -# ./config.sh --url https://github.com/cactus-compute/cactus --token -# 3. Accept defaults for runner group, name, and labels (it auto-detects self-hosted, linux, ARM64). -# 4. Start as service: -# sudo ./svc.sh install -# sudo ./svc.sh start -# 5. Add repo secret: Go to the target repo → Settings → Secrets and variables → Actions → New repository secret → Name: HF_TOKEN, Value: -# 6. Verify: The runner should appear as "Online" in the repo's Settings → Actions → Runners. Trigger the workflow from Actions → PR Verification → Run workflow. - -on: - pull_request_review: - types: [submitted] - workflow_dispatch: - -jobs: - gate: - runs-on: ubuntu-latest - outputs: - approved: ${{ steps.check.outputs.approved }} - steps: - - id: check - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "approved=true" >> $GITHUB_OUTPUT - elif [ "${{ github.event.review.state }}" = "approved" ]; then - echo "approved=true" >> $GITHUB_OUTPUT - else - echo "approved=false" >> $GITHUB_OUTPUT - fi - - build: - needs: gate - if: needs.gate.outputs.approved == 'true' - runs-on: [self-hosted, linux, arm64] - steps: - - uses: actions/checkout@v4 - with: - clean: false - - - name: Check for Python changes - id: check_python - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "python_changed=false" >> $GITHUB_OUTPUT - else - CHANGED=$(gh pr diff ${{ github.event.pull_request.number }} --name-only | grep -cE '^python/(src/|requirements\.txt)' || true) - echo "python_changed=$([[ $CHANGED -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT - fi - - - name: Ensure venv exists - run: | - if [ ! -f venv/bin/activate ]; then - echo "No venv found, running setup..." - source ./setup - fi - - - name: Rebuild venv if Python sources changed - if: steps.check_python.outputs.python_changed == 'true' - run: | - deactivate || true - rm -rf venv - source ./setup - - - name: Purge cached weights if converters changed - if: steps.check_python.outputs.python_changed == 'true' - run: rm -rf weights/ - - - name: Convert models if weights missing - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - source venv/bin/activate - convert_if_missing() { - local model_id=$1 output_dir=$2 precision=$3 - if [ ! -f "weights/$output_dir/config.txt" ]; then - echo "Converting $model_id -> weights/$output_dir" - python3 -m src.cli convert "$model_id" "weights/$output_dir" --precision "$precision" ${HF_TOKEN:+--token "$HF_TOKEN"} || echo "WARNING: conversion of $model_id failed (exit $?), checking fallback directory" - local default_dir="weights/$(echo "$model_id" | sed 's|.*/||' | tr '[:upper:]' '[:lower:]')" - if [ ! -f "weights/$output_dir/config.txt" ] && [ -f "$default_dir/config.txt" ]; then - mv "$default_dir" "weights/$output_dir" - fi - fi - } - convert_if_missing "Qwen/Qwen3-0.6B" "qwen3-0.6b" "INT8" - convert_if_missing "nomic-ai/nomic-embed-text-v2-moe" "nomic-embed-text-v2-moe" "INT8" - convert_if_missing "LiquidAI/LFM2.5-350M" "lfm2-350m" "INT8" - convert_if_missing "LiquidAI/LFM2-8B-A1B" "lfm2-8b-a1b" "INT8" - convert_if_missing "google/gemma-3-270m-it" "gemma-3-270m-it" "INT8" - convert_if_missing "openai/whisper-small" "whisper-small" "INT8" - convert_if_missing "UsefulSensors/moonshine-base" "moonshine-base" "INT8" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int8" "INT8" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-int4" "INT4" - convert_if_missing "LiquidAI/LFM2-VL-450M" "lfm2-vl-450m-fp16" "FP16" - - - name: Build library - run: bash cactus/build.sh - - - name: Build tests - run: | - cd tests - rm -rf build && mkdir -p build && cd build - cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF - make -j$(nproc) - - unit-tests: - needs: build - runs-on: [self-hosted, linux, arm64] - steps: - - uses: actions/checkout@v4 - with: - clean: false - - name: Run unit tests - env: - CACTUS_INDEX_PATH: ${{ runner.temp }}/cactus_index - CACTUS_NO_CLOUD_TELE: "1" - run: | - mkdir -p "$CACTUS_INDEX_PATH" - FAILED=0 - for test in test_graph test_kernel test_kv_cache test_index test_model_loading; do - echo "=== Running $test ===" - if ! ./tests/build/$test; then - FAILED=$((FAILED + 1)) - fi - done - if [ $FAILED -gt 0 ]; then exit 1; fi - - exhaustive-tests: - needs: build - runs-on: [self-hosted, linux, arm64] - strategy: - fail-fast: false - matrix: - include: - - family: qwen - model_dir: qwen3-0.6b - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: nomic - model_dir: nomic-embed-text-v2-moe - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: lfm2 - model_dir: lfm2-350m - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: lfm2moe - model_dir: lfm2-8b-a1b - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: gemma - model_dir: gemma-3-270m-it - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: whisper - model_dir: whisper-small - env_var: CACTUS_TEST_TRANSCRIBE_MODEL - precision: INT8 - - family: moonshine - model_dir: moonshine-base - env_var: CACTUS_TEST_TRANSCRIBE_MODEL - precision: INT8 - - family: lfm2vl - model_dir: lfm2-vl-450m-int8 - env_var: CACTUS_TEST_MODEL - precision: INT8 - - family: lfm2vl - model_dir: lfm2-vl-450m-int4 - env_var: CACTUS_TEST_MODEL - precision: INT4 - - family: lfm2vl - model_dir: lfm2-vl-450m-fp16 - env_var: CACTUS_TEST_MODEL - precision: FP16 - steps: - - uses: actions/checkout@v4 - with: - clean: false - - name: Run exhaustive test - env: - CACTUS_NO_CLOUD_TELE: "1" - run: | - export ${{ matrix.env_var }}="${{ github.workspace }}/weights/${{ matrix.model_dir }}" - export CACTUS_TEST_ASSETS="${{ github.workspace }}/tests/assets" - export CACTUS_TEST_GOLDEN_FILE="${{ github.workspace }}/tests/golden/golden.json" - export CACTUS_TEST_GOLDEN_FAMILY="${{ matrix.family }}" - export CACTUS_TEST_GOLDEN_PRECISION="${{ matrix.precision }}" - if [ "${{ matrix.family }}" = "whisper" ] || [ "${{ matrix.family }}" = "moonshine" ]; then - export CACTUS_TEST_VAD_MODEL="${{ github.workspace }}/weights/silero-vad" - fi - ./tests/build/test_exhaustive diff --git a/.gitignore b/.gitignore index e0918fde3..743d97d84 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,8 @@ **/build/ -**/build_inc/ *.a *.so *.dylib *.bin -*.gguf -*.npy *.cg !cactus-engine/libs/curl/**/*.a @@ -13,40 +10,21 @@ apple/build-*/ apple/*.xcframework -apple/libcactus*.a -android/libcactus.so -android/libcactus.a -android/libs/ -android/arm64-v8a/ -bindings/flutter/libcactus.so -bindings/flutter/*.xcframework .claude .vscode .DS_Store -__pycache__ __pycache__/ *.pyc *.egg-info +.pytest_cache/ venv weights/ -stage/ +transpiled/ python/dist/ python/cactus/bin/ python/cactus/bindings/lib/ -.cache/ -tmp site/ site_docs/ - -notes.txt -profile.txt -*_profile.txt -tests/plan.md -tests/assets/test_wav_info.txt -vision_activations.json -gemma-4-eap-extras/ -.githooks/ -ruy diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29bb..000000000 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bfff37112..36b5fab1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ Thank you for your interest in contributing to Cactus! This document covers the 1. Fork the repository and create a branch from `main`. 2. Make your changes, keeping the scope focused on the relevant GitHub issue. 3. Run `cactus test` to verify your changes build and pass all tests. -4. Run `cactus test --performance` if your changes affect performance-critical paths. +4. Run `cactus test --suite performance` if your changes affect performance-critical paths. 5. Update documentation if your changes affect the public API or user-facing behavior. 6. Submit a pull request with a clear description of what you changed and why. @@ -41,10 +41,10 @@ cactus test --ios cactus test --android # Run performance tests -cactus test --performance +cactus test --suite performance # Test a specific model -cactus test --model LiquidAI/LFM2.5-1.2B-Instruct +cactus test --model LiquidAI/LFM2-VL-450M ``` ## Developer Certificate of Origin diff --git a/README.md b/README.md index 11a119144..979c43976 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ A low-latency AI engine for mobile devices & wearables. ## Cactus Engine ```cpp -#include "cactus.h" +#include "cactus_engine.h" cactus_model_t model = cactus_init( "path/to/weight/folder", @@ -102,7 +102,7 @@ Example response from Gemma3-270m ## Cactus Graph ```cpp -#include "cactus.h" +#include "cactus_graph.h" CactusGraph graph; auto a = graph.input({2, 3}, Precision::FP16); @@ -227,67 +227,84 @@ cactus build # default static lib ## Using this repo ``` -┌──────────────────────────────────────────────────────────────────────────────┐ -│ │ -│ Step 0: if on Linux (Ubuntu/Debian) │ -│ sudo apt-get install python3 python3-venv python3-pip cmake │ -│ build-essential libcurl4-openssl-dev │ -│ │ -│ Step 1: clone and setup │ -│ git clone https://github.com/cactus-compute/cactus && cd cactus │ -│ source ./setup │ -│ │ -│ Step 2: use the commands │ -│──────────────────────────────────────────────────────────────────────────────│ -│ │ -│ cactus auth manage Cloud API key │ -│ --status show key status │ -│ --clear remove saved key │ -│ │ -│ cactus run [model] chat playground (gemma-4-E2B-it) │ -│ --image image file for VLM inference │ -│ --audio audio file (WAV) for audio chat │ -│ --system system prompt │ -│ --prompt send prompt immediately │ -│ --thinking enable thinking/reasoning mode │ -│ --token HF token (gated models) │ -│ --reconvert force reconversion from source │ -│ │ -│ cactus transcribe [model] speech-to-text (parakeet-tdt-0.6b-v3) │ -│ --file audio file to transcribe (required) │ -│ --language language code (default: en) │ -│ --token HF token (gated models) │ -│ --reconvert force reconversion from source │ -│ │ -│ cactus download fetch pre-converted CQ from Cactus-Compute │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --token HuggingFace API token │ -│ │ -│ cactus convert [dir] convert model to CQ format │ -│ (pre-converted if available, else │ -│ built from source) │ -│ --bits 1|2|3|4 CQ quantization (default: 4) │ -│ --token HuggingFace API token │ -│ --reconvert force build from source │ -│ │ -│ cactus build build for ARM → build/libcactus.a │ -│ --apple Apple (iOS/macOS) │ -│ --android Android │ -│ --python shared lib for Python FFI │ -│ │ -│ cactus test run unit tests and benchmarks │ -│ --model default: google/gemma-4-E2B-it │ -│ --suite run a specific suite (llm, vlm, stt, │ -│ embed, rag, graph, index, kernel, etc.) │ -│ --token HuggingFace API token │ -│ --reconvert force reconversion from source │ -│ --ios run on connected iPhone │ -│ --android run on connected Android │ -│ │ -│ cactus clean remove all build artifacts │ -│ cactus --help show all commands and flags │ -│ │ -└──────────────────────────────────────────────────────────────────────────────┘ +┌────────────────────────────────────────────────────────────────────────────────┐ +│ │ +│ Step 0: if on Linux (Ubuntu/Debian) │ +│ sudo apt-get install python3 python3-venv python3-pip cmake │ +│ build-essential libcurl4-openssl-dev │ +│ │ +│ Step 1: clone and setup │ +│ git clone https://github.com/cactus-compute/cactus && cd cactus │ +│ source ./setup │ +│ │ +│ Step 2: use the commands │ +│────────────────────────────────────────────────────────────────────────────────│ +│ │ +│ cactus auth manage cloud API key │ +│ --status show key status │ +│ --clear remove saved key │ +│ │ +│ cactus run run a model (downloads if needed) │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform cpu|apple target accelerator (default: cpu) │ +│ --image image file for VLM inference │ +│ --audio audio file for audio chat │ +│ --system system prompt │ +│ --prompt send prompt immediately │ +│ --thinking enable thinking/reasoning mode │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force local convert+transpile fallback │ +│ │ +│ cactus transcribe [model] transcribe audio with a model │ +│ --file audio file to transcribe (required) │ +│ --language language code (default: en) │ +│ --token HuggingFace token (gated models) │ +│ --reconvert force reconversion from source │ +│ │ +│ cactus download download a pre-built bundle │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --platform cpu|apple target accelerator (default: cpu) │ +│ --token HuggingFace token │ +│ │ +│ cactus convert [dir] convert HuggingFace weights to CQ │ +│ --bits 1|2|3|4 CQ quantization (default: 4) │ +│ --token HuggingFace token │ +│ --reconvert force build from source │ +│ --lora merge a LoRA adapter before converting │ +│ │ +│ cactus transpile build a runnable bundle from CQ weights │ +│ --weights-dir path to CQ weights (default: lookup) │ +│ --task force task type (default: auto) │ +│ --artifact-dir bundle output (default: weights/) │ +│ │ +│ cactus serve [model] OpenAI-compatible local HTTP server │ +│ --host bind address (default: 127.0.0.1) │ +│ --port port (default: 8080) │ +│ │ +│ cactus list list local converted weights and bundles │ +│ │ +│ cactus build build cactus libraries │ +│ --apple Apple (iOS/macOS) │ +│ --android Android │ +│ --python shared lib for Python FFI │ +│ │ +│ cactus test run the test suite │ +│ --component kernels | graph | engine | all │ +│ (default: all) │ +│ --model default: LiquidAI/LFM2-VL-450M │ +│ --transcription-model default: openai/whisper-base │ +│ --suite run a single test suite (resolved │ +│ across components; e.g. performance │ +│ → kernels + graph, llm → engine) │ +│ --list list components and suites │ +│ --ios run on connected iPhone │ +│ --android run on connected Android │ +│ --enable-telemetry send cloud telemetry (off by default) │ +│ │ +│ cactus clean delete build artifacts │ +│ cactus --help show this help │ +│ │ +└────────────────────────────────────────────────────────────────────────────────┘ ``` ## Maintaining Organisations diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index d04de1768..bf1165792 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -8,13 +8,8 @@ if(NOT ${ANDROID_ABI} STREQUAL "arm64-v8a") message(FATAL_ERROR "Unsupported Android ABI: ${ANDROID_ABI}. Cactus only supports arm64-v8a.") endif() -set(CACTUS_MBEDTLS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/mbedtls/${ANDROID_ABI}") -if(NOT EXISTS "${CACTUS_MBEDTLS_ROOT}") - set(CACTUS_MBEDTLS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../libs/mbedtls/android/${ANDROID_ABI}") -endif() - if(NOT TARGET cactus_engine) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_BINARY_DIR}/cactus) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine ${CMAKE_CURRENT_BINARY_DIR}/cactus-engine) endif() set(_DEPS cactus_engine cactus_graph cactus_kernels) @@ -22,31 +17,18 @@ set(_DEPS cactus_engine cactus_graph cactus_kernels) add_library(cactus_jni SHARED ${CMAKE_CURRENT_SOURCE_DIR}/cactus_jni.cpp) set_target_properties(cactus_jni PROPERTIES - OUTPUT_NAME cactus + OUTPUT_NAME cactus_engine CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON ) target_include_directories(cactus_jni PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine/src ) target_link_libraries(cactus_jni PRIVATE ${_DEPS}) -find_library(LOG_LIB log) -target_link_libraries(cactus_jni PRIVATE ${LOG_LIB} android) - -set(_MBEDTLS "${CACTUS_MBEDTLS_ROOT}/lib/libmbedtls.a") -set(_MBEDX509 "${CACTUS_MBEDTLS_ROOT}/lib/libmbedx509.a") -set(_MBEDCRYPTO "${CACTUS_MBEDTLS_ROOT}/lib/libmbedcrypto.a") -if(EXISTS "${_MBEDTLS}" AND EXISTS "${_MBEDX509}" AND EXISTS "${_MBEDCRYPTO}") - target_link_libraries(cactus_jni PRIVATE "${_MBEDTLS}" "${_MBEDX509}" "${_MBEDCRYPTO}") -endif() - -target_compile_definitions(cactus_jni PRIVATE PLATFORM_CPU_ONLY=1) - set(_CACTUS_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../CACTUS_VERSION") if(EXISTS "${_CACTUS_VERSION_FILE}") file(READ "${_CACTUS_VERSION_FILE}" _CACTUS_VERSION_CONTENT) @@ -59,4 +41,4 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug") endif() target_compile_options(cactus_jni PRIVATE -fvisibility=hidden -fvisibility-inlines-hidden -ffunction-sections -fdata-sections) -target_link_options(cactus_jni PRIVATE -Wl,--gc-sections -Wl,--exclude-libs,ALL -flto -Wl,-z,max-page-size=16384) +target_link_options(cactus_jni PRIVATE -Wl,--gc-sections -flto -Wl,-z,max-page-size=16384) diff --git a/android/Cactus.android.kt b/android/Cactus.android.kt deleted file mode 100644 index 2105f5c46..000000000 --- a/android/Cactus.android.kt +++ /dev/null @@ -1,115 +0,0 @@ -@file:JvmName("Cactus") -package com.cactus - -private object CactusJNI { - init { - System.loadLibrary("cactus") - nativeSetFramework() - } - - @JvmStatic external fun nativeSetFramework() - @JvmStatic external fun nativeGetLastError(): String - @JvmStatic external fun nativeSetCacheDir(cacheDir: String) - @JvmStatic external fun nativeSetAppId(appId: String) - @JvmStatic external fun nativeTelemetryFlush() - @JvmStatic external fun nativeTelemetryShutdown() - @JvmStatic external fun nativeInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long - @JvmStatic external fun nativeDestroy(handle: Long) - @JvmStatic external fun nativeReset(handle: Long) - @JvmStatic external fun nativeStop(handle: Long) - @JvmStatic external fun nativeComplete(handle: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String - @JvmStatic external fun nativePrefill(handle: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeDetectLanguage(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeLogSetLevel(level: Int) - @JvmStatic external fun nativeLogSetCallback(callback: CactusLogCallback?) - @JvmStatic external fun nativeTranscribe(handle: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String - @JvmStatic external fun nativeEmbed(handle: Long, text: String, normalize: Boolean): FloatArray - @JvmStatic external fun nativeRagQuery(handle: Long, query: String, topK: Int): String - @JvmStatic external fun nativeTokenize(handle: Long, text: String): IntArray - @JvmStatic external fun nativeScoreWindow(handle: Long, tokens: IntArray, start: Int, end: Int, context: Int): String - @JvmStatic external fun nativeImageEmbed(handle: Long, imagePath: String): FloatArray - @JvmStatic external fun nativeAudioEmbed(handle: Long, audioPath: String): FloatArray - @JvmStatic external fun nativeVad(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeDiarize(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeEmbedSpeaker(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray?): String - @JvmStatic external fun nativeStreamTranscribeInit(handle: Long, optionsJson: String?): Long - @JvmStatic external fun nativeStreamTranscribeProcess(handle: Long, pcmData: ByteArray): String - @JvmStatic external fun nativeStreamTranscribeStop(handle: Long): String - @JvmStatic external fun nativeIndexInit(indexDir: String, embeddingDim: Int): Long - @JvmStatic external fun nativeIndexAdd(handle: Long, ids: IntArray, documents: Array, metadatas: Array?, embeddings: Array, embeddingDim: Long): Int - @JvmStatic external fun nativeIndexDelete(handle: Long, ids: IntArray): Int - @JvmStatic external fun nativeIndexGet(handle: Long, ids: IntArray): String - @JvmStatic external fun nativeIndexQuery(handle: Long, embedding: FloatArray, topK: Long, optionsJson: String?): String - @JvmStatic external fun nativeIndexCompact(handle: Long): Int - @JvmStatic external fun nativeIndexDestroy(handle: Long) -} - -actual fun cactusInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long { - val h = CactusJNI.nativeInit(modelPath, corpusDir, cacheIndex) - if (h == 0L) throw RuntimeException(CactusJNI.nativeGetLastError().ifEmpty { "Failed to initialize model" }) - return h -} -actual fun cactusDestroy(model: Long) = CactusJNI.nativeDestroy(model) -actual fun cactusReset(model: Long) = CactusJNI.nativeReset(model) -actual fun cactusStop(model: Long) = CactusJNI.nativeStop(model) -actual fun cactusGetLastError(): String = CactusJNI.nativeGetLastError() -actual fun cactusSetTelemetryEnvironment(cacheDir: String) = CactusJNI.nativeSetCacheDir(cacheDir) -actual fun cactusSetAppId(appId: String) = CactusJNI.nativeSetAppId(appId) -actual fun cactusTelemetryFlush() = CactusJNI.nativeTelemetryFlush() -actual fun cactusTelemetryShutdown() = CactusJNI.nativeTelemetryShutdown() -actual fun cactusComplete(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String = - CactusJNI.nativeComplete(model, messagesJson, optionsJson, toolsJson, callback, pcmData) -actual fun cactusPrefill(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativePrefill(model, messagesJson, optionsJson, toolsJson, pcmData) -actual fun cactusTranscribe(model: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String = - CactusJNI.nativeTranscribe(model, audioPath, prompt, optionsJson, callback, pcmData) -actual fun cactusEmbed(model: Long, text: String, normalize: Boolean): FloatArray = - CactusJNI.nativeEmbed(model, text, normalize) -actual fun cactusImageEmbed(model: Long, imagePath: String): FloatArray = - CactusJNI.nativeImageEmbed(model, imagePath) -actual fun cactusAudioEmbed(model: Long, audioPath: String): FloatArray = - CactusJNI.nativeAudioEmbed(model, audioPath) -actual fun cactusVad(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeVad(model, audioPath, optionsJson, pcmData) -actual fun cactusDiarize(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeDiarize(model, audioPath, optionsJson, pcmData) -actual fun cactusEmbedSpeaker(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray?): String = - CactusJNI.nativeEmbedSpeaker(model, audioPath, optionsJson, pcmData, maskWeights) -actual fun cactusRagQuery(model: Long, query: String, topK: Int): String = - CactusJNI.nativeRagQuery(model, query, topK) -actual fun cactusTokenize(model: Long, text: String): IntArray = - CactusJNI.nativeTokenize(model, text) -actual fun cactusScoreWindow(model: Long, tokens: IntArray, start: Int, end: Int, context: Int): String = - CactusJNI.nativeScoreWindow(model, tokens, start, end, context) -actual fun cactusStreamTranscribeStart(model: Long, optionsJson: String?): Long { - val h = CactusJNI.nativeStreamTranscribeInit(model, optionsJson) - if (h == 0L) throw RuntimeException(CactusJNI.nativeGetLastError().ifEmpty { "Failed to create stream transcriber" }) - return h -} -actual fun cactusStreamTranscribeProcess(stream: Long, pcmData: ByteArray): String = - CactusJNI.nativeStreamTranscribeProcess(stream, pcmData) -actual fun cactusStreamTranscribeStop(stream: Long): String = - CactusJNI.nativeStreamTranscribeStop(stream) -actual fun cactusIndexInit(indexDir: String, embeddingDim: Int): Long { - val h = CactusJNI.nativeIndexInit(indexDir, embeddingDim) - if (h == 0L) throw RuntimeException("Failed to initialize index") - return h -} -actual fun cactusIndexAdd(index: Long, ids: IntArray, documents: Array, embeddings: Array, metadatas: Array?): Int = - CactusJNI.nativeIndexAdd(index, ids, documents, metadatas, embeddings, embeddings[0].size.toLong()) -actual fun cactusIndexDelete(index: Long, ids: IntArray): Int = - CactusJNI.nativeIndexDelete(index, ids) -actual fun cactusIndexGet(index: Long, ids: IntArray): String = - CactusJNI.nativeIndexGet(index, ids) -actual fun cactusIndexQuery(index: Long, embedding: FloatArray, optionsJson: String?): String = - CactusJNI.nativeIndexQuery(index, embedding, 1000L, optionsJson) -actual fun cactusIndexCompact(index: Long): Int = - CactusJNI.nativeIndexCompact(index) -actual fun cactusIndexDestroy(index: Long) = - CactusJNI.nativeIndexDestroy(index) -actual fun cactusDetectLanguage(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeDetectLanguage(model, audioPath, optionsJson, pcmData) -actual fun cactusLogSetLevel(level: Int) = - CactusJNI.nativeLogSetLevel(level) -actual fun cactusLogSetCallback(callback: CactusLogCallback?) = - CactusJNI.nativeLogSetCallback(callback) diff --git a/android/Cactus.common.kt b/android/Cactus.common.kt deleted file mode 100644 index 51bbf0609..000000000 --- a/android/Cactus.common.kt +++ /dev/null @@ -1,44 +0,0 @@ -package com.cactus - -fun interface CactusTokenCallback { - fun onToken(token: String, tokenId: Int) -} - -fun interface CactusLogCallback { - fun onLog(level: Int, component: String, message: String) -} - -expect fun cactusInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long -expect fun cactusDestroy(model: Long) -expect fun cactusReset(model: Long) -expect fun cactusStop(model: Long) -expect fun cactusGetLastError(): String -expect fun cactusSetTelemetryEnvironment(cacheDir: String) -expect fun cactusSetAppId(appId: String) -expect fun cactusTelemetryFlush() -expect fun cactusTelemetryShutdown() -expect fun cactusComplete(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray? = null): String -expect fun cactusPrefill(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray? = null): String -expect fun cactusTranscribe(model: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String -expect fun cactusEmbed(model: Long, text: String, normalize: Boolean): FloatArray -expect fun cactusImageEmbed(model: Long, imagePath: String): FloatArray -expect fun cactusAudioEmbed(model: Long, audioPath: String): FloatArray -expect fun cactusVad(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String -expect fun cactusRagQuery(model: Long, query: String, topK: Int): String -expect fun cactusTokenize(model: Long, text: String): IntArray -expect fun cactusScoreWindow(model: Long, tokens: IntArray, start: Int, end: Int, context: Int): String -expect fun cactusStreamTranscribeStart(model: Long, optionsJson: String?): Long -expect fun cactusStreamTranscribeProcess(stream: Long, pcmData: ByteArray): String -expect fun cactusStreamTranscribeStop(stream: Long): String -expect fun cactusIndexInit(indexDir: String, embeddingDim: Int): Long -expect fun cactusIndexAdd(index: Long, ids: IntArray, documents: Array, embeddings: Array, metadatas: Array?): Int -expect fun cactusIndexDelete(index: Long, ids: IntArray): Int -expect fun cactusIndexGet(index: Long, ids: IntArray): String -expect fun cactusIndexQuery(index: Long, embedding: FloatArray, optionsJson: String?): String -expect fun cactusIndexCompact(index: Long): Int -expect fun cactusIndexDestroy(index: Long) -expect fun cactusDetectLanguage(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String -expect fun cactusDiarize(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String -expect fun cactusEmbedSpeaker(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray? = null): String -expect fun cactusLogSetLevel(level: Int) -expect fun cactusLogSetCallback(callback: CactusLogCallback?) diff --git a/android/Cactus.ios.kt b/android/Cactus.ios.kt deleted file mode 100644 index 22fbd2947..000000000 --- a/android/Cactus.ios.kt +++ /dev/null @@ -1,517 +0,0 @@ -package com.cactus - -import cactus.* -import kotlinx.cinterop.* - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long { - val ptr = cactus_init(modelPath, corpusDir, cacheIndex) - ?: throw RuntimeException(cactus_get_last_error()?.toKString() ?: "Failed to initialize model") - return ptr.rawValue.toLong() -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusDestroy(model: Long) { - cactus_destroy(model.toCPointer()) -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusReset(model: Long) { cactus_reset(model.toCPointer()) } - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusStop(model: Long) { cactus_stop(model.toCPointer()) } - -actual fun cactusGetLastError(): String = cactus_get_last_error()?.toKString() ?: "" - -actual fun cactusSetTelemetryEnvironment(cacheDir: String) { - cactus_set_telemetry_environment(null, cacheDir, null) -} - -actual fun cactusSetAppId(appId: String) { cactus_set_app_id(appId) } - -actual fun cactusTelemetryFlush() { cactus_telemetry_flush() } - -actual fun cactusTelemetryShutdown() { cactus_telemetry_shutdown() } - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusComplete(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val callbackRef = callback?.let { StableRef.create(it) } - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - try { - val result = cactus_complete( - model.toCPointer(), - messagesJson, - buffer, - 65536u, - optionsJson, - toolsJson, - callbackRef?.let { - staticCFunction?, UInt, COpaquePointer?, Unit> { token, tokenId, userData -> - if (token != null && userData != null) { - userData.asStableRef().get().onToken(token.toKString(), tokenId.toInt()) - } - } - }, - callbackRef?.asCPointer(), - pcmPtr?.reinterpret(), - pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } finally { - callbackRef?.dispose() - } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusPrefill(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - val result = cactus_prefill( - model.toCPointer(), - messagesJson, - buffer, - 65536u, - optionsJson, - toolsJson, - pcmPtr?.reinterpret(), - pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusTranscribe(model: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val callbackRef = callback?.let { StableRef.create(it) } - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - try { - val result = cactus_transcribe( - model.toCPointer(), - audioPath, - prompt, - buffer, - 65536u, - optionsJson, - callbackRef?.let { - staticCFunction?, UInt, COpaquePointer?, Unit> { token, tokenId, userData -> - if (token != null && userData != null) { - userData.asStableRef().get().onToken(token.toKString(), tokenId.toInt()) - } - } - }, - callbackRef?.asCPointer(), - pcmPtr?.reinterpret(), - pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } finally { - callbackRef?.dispose() - } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusEmbed(model: Long, text: String, normalize: Boolean): FloatArray { - memScoped { - val buffer = allocArray(4096) - val dimPtr = alloc() - val result = cactus_embed(model.toCPointer(), text, buffer, 4096u, dimPtr.ptr, normalize) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return FloatArray(dimPtr.value.toInt()) { buffer[it] } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusImageEmbed(model: Long, imagePath: String): FloatArray { - memScoped { - val buffer = allocArray(4096) - val dimPtr = alloc() - val result = cactus_image_embed(model.toCPointer(), imagePath, buffer, 4096u, dimPtr.ptr) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return FloatArray(dimPtr.value.toInt()) { buffer[it] } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusAudioEmbed(model: Long, audioPath: String): FloatArray { - memScoped { - val buffer = allocArray(4096) - val dimPtr = alloc() - val result = cactus_audio_embed(model.toCPointer(), audioPath, buffer, 4096u, dimPtr.ptr) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return FloatArray(dimPtr.value.toInt()) { buffer[it] } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusVad(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - val result = cactus_vad( - model.toCPointer(), audioPath, buffer, 65536u, optionsJson, - pcmPtr?.reinterpret(), pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusRagQuery(model: Long, query: String, topK: Int): String { - memScoped { - val buffer = allocArray(65536) - val result = cactus_rag_query(model.toCPointer(), query, buffer, 65536u, topK.toULong()) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusTokenize(model: Long, text: String): IntArray { - memScoped { - val buffer = allocArray(8192) - val tokenLen = alloc() - val result = cactus_tokenize(model.toCPointer(), text, buffer, 8192u, tokenLen.ptr) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return IntArray(tokenLen.value.toInt()) { buffer[it].toInt() } - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusScoreWindow(model: Long, tokens: IntArray, start: Int, end: Int, context: Int): String { - memScoped { - val buffer = allocArray(65536) - val tokenBuffer = allocArray(tokens.size) - tokens.forEachIndexed { i, v -> tokenBuffer[i] = v.toUInt() } - val result = cactus_score_window( - model.toCPointer(), tokenBuffer, tokens.size.toULong(), - start.toULong(), end.toULong(), context.toULong(), - buffer, 65536u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusStreamTranscribeStart(model: Long, optionsJson: String?): Long { - val ptr = cactus_stream_transcribe_start(model.toCPointer(), optionsJson) - ?: throw RuntimeException(cactus_get_last_error()?.toKString() ?: "Failed to create stream transcriber") - return ptr.rawValue.toLong() -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusStreamTranscribeProcess(stream: Long, pcmData: ByteArray): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData.refTo(0).getPointer(this) - val result = cactus_stream_transcribe_process( - stream.toCPointer(), pcmPtr.reinterpret(), pcmData.size.toULong(), buffer, 65536u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusStreamTranscribeStop(stream: Long): String { - memScoped { - val buffer = allocArray(65536) - val result = cactus_stream_transcribe_stop(stream.toCPointer(), buffer, 65536u) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexInit(indexDir: String, embeddingDim: Int): Long { - val ptr = cactus_index_init(indexDir, embeddingDim.toULong()) - ?: throw RuntimeException("Failed to initialize index") - return ptr.rawValue.toLong() -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexAdd(index: Long, ids: IntArray, documents: Array, embeddings: Array, metadatas: Array?): Int { - memScoped { - val idPtr = allocArray(ids.size) - ids.forEachIndexed { i, v -> idPtr[i] = v } - val docPtrs = allocArray>(documents.size) - documents.forEachIndexed { i, doc -> docPtrs[i] = doc.cstr.ptr } - val metaPtrs = metadatas?.let { - val ptrs = allocArray>(it.size) - it.forEachIndexed { i, meta -> ptrs[i] = meta.cstr.ptr } - ptrs - } - val embPtrs = allocArray>(embeddings.size) - embeddings.forEachIndexed { i, emb -> - val embArr = allocArray(emb.size) - emb.forEachIndexed { j, v -> embArr[j] = v } - embPtrs[i] = embArr - } - val result = cactus_index_add( - index.toCPointer(), idPtr, docPtrs, metaPtrs, embPtrs, - ids.size.toULong(), embeddings[0].size.toULong() - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return result - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexDelete(index: Long, ids: IntArray): Int { - memScoped { - val idPtr = allocArray(ids.size) - ids.forEachIndexed { i, v -> idPtr[i] = v } - val result = cactus_index_delete(index.toCPointer(), idPtr, ids.size.toULong()) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return result - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexGet(index: Long, ids: IntArray): String { - memScoped { - val count = ids.size - val idPtr = allocArray(count) - ids.forEachIndexed { i, v -> idPtr[i] = v } - - val docBufSize = 4096 - val embBufSize = 4096 - - val docRaw = Array(count) { allocArray(docBufSize) } - val metaRaw = Array(count) { allocArray(docBufSize) } - val embRaw = Array(count) { allocArray(embBufSize) } - - val docBuffers = allocArray>(count) - val docBufferSizes = allocArray(count) - val metaBuffers = allocArray>(count) - val metaBufferSizes = allocArray(count) - val embBuffers = allocArray>(count) - val embBufferSizes = allocArray(count) - - for (i in 0 until count) { - docBuffers[i] = docRaw[i] - docBufferSizes[i] = docBufSize.toULong() - metaBuffers[i] = metaRaw[i] - metaBufferSizes[i] = docBufSize.toULong() - embBuffers[i] = embRaw[i] - embBufferSizes[i] = embBufSize.toULong() - } - - val result = cactus_index_get( - index.toCPointer(), idPtr, count.toULong(), - docBuffers, docBufferSizes, metaBuffers, metaBufferSizes, - embBuffers, embBufferSizes - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - val sb = StringBuilder("{\"results\":[") - for (i in 0 until count) { - if (i > 0) sb.append(",") - sb.append("{\"document\":\"${docRaw[i].toKString()}\"") - val meta = metaRaw[i].toKString() - if (meta.isNotEmpty()) sb.append(",\"metadata\":\"$meta\"") else sb.append(",\"metadata\":null") - sb.append(",\"embedding\":[") - val embDim = embBufferSizes[i].toInt() - for (j in 0 until embDim) { - if (j > 0) sb.append(",") - sb.append(embRaw[i][j]) - } - sb.append("]}") - } - sb.append("]}") - return sb.toString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexQuery(index: Long, embedding: FloatArray, optionsJson: String?): String { - val topK = 1000L - memScoped { - val embArr = allocArray(embedding.size) - embedding.forEachIndexed { i, v -> embArr[i] = v } - val embPtr = alloc>() - embPtr.value = embArr - val idBuffer = allocArray(topK) - val scoreBuffer = allocArray(topK) - val idBufferSize = alloc() - val scoreBufferSize = alloc() - idBufferSize.value = topK.toULong() - scoreBufferSize.value = topK.toULong() - val idPtrPtr = alloc>() - idPtrPtr.value = idBuffer - val scorePtrPtr = alloc>() - scorePtrPtr.value = scoreBuffer - val result = cactus_index_query( - index.toCPointer(), embPtr.ptr, 1u, embedding.size.toULong(), optionsJson, - idPtrPtr.ptr, idBufferSize.ptr, scorePtrPtr.ptr, scoreBufferSize.ptr - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - val sb = StringBuilder("{\"results\":[") - for (i in 0 until idBufferSize.value.toInt()) { - if (i > 0) sb.append(",") - sb.append("{\"id\":${idBuffer[i]},\"score\":${scoreBuffer[i]}}") - } - sb.append("]}") - return sb.toString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexCompact(index: Long): Int { - val result = cactus_index_compact(index.toCPointer()) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return result -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusIndexDestroy(index: Long) { - cactus_index_destroy(index.toCPointer()) -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusDetectLanguage(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - val result = cactus_detect_language( - model.toCPointer(), audioPath, buffer, 65536u, optionsJson, - pcmPtr?.reinterpret(), pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusDiarize(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - val result = cactus_diarize( - model.toCPointer(), audioPath, buffer, 65536u, optionsJson, - pcmPtr?.reinterpret(), pcmData?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusEmbedSpeaker(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray?): String { - memScoped { - val buffer = allocArray(65536) - val pcmPtr = pcmData?.refTo(0)?.getPointer(this) - val maskPtr = maskWeights?.let { - val arr = allocArray(it.size) - it.forEachIndexed { i, v -> arr[i] = v } - arr - } - val result = cactus_embed_speaker( - model.toCPointer(), audioPath, buffer, 65536u, optionsJson, - pcmPtr?.reinterpret(), pcmData?.size?.toULong() ?: 0u, - maskPtr, maskWeights?.size?.toULong() ?: 0u - ) - if (result < 0) { - val error = cactus_get_last_error()?.toKString() ?: "Unknown error" - throw RuntimeException(error) - } - return buffer.toKString() - } -} - -actual fun cactusLogSetLevel(level: Int) { - cactus_log_set_level(level) -} - -private var _logCallbackRef: StableRef? = null - -@OptIn(ExperimentalForeignApi::class) -actual fun cactusLogSetCallback(callback: CactusLogCallback?) { - _logCallbackRef?.dispose() - _logCallbackRef = null - if (callback == null) { - cactus_log_set_callback(null, null) - } else { - val callbackRef = StableRef.create(callback) - _logCallbackRef = callbackRef - cactus_log_set_callback( - staticCFunction?, CPointer?, COpaquePointer?, Unit> { level, component, message, userData -> - if (userData != null) { - userData.asStableRef().get().onLog( - level, - component?.toKString() ?: "", - message?.toKString() ?: "" - ) - } - }, - callbackRef.asCPointer() - ) - } -} diff --git a/android/Cactus.kt b/android/Cactus.kt deleted file mode 100644 index 5614e0302..000000000 --- a/android/Cactus.kt +++ /dev/null @@ -1,122 +0,0 @@ -package com.cactus - -fun interface CactusTokenCallback { - fun onToken(token: String, tokenId: Int) -} - -fun interface CactusLogCallback { - fun onLog(level: Int, component: String, message: String) -} - -private object CactusJNI { - init { - System.loadLibrary("cactus") - nativeSetFramework() - } - - @JvmStatic external fun nativeSetFramework() - @JvmStatic external fun nativeGetLastError(): String - @JvmStatic external fun nativeSetCacheDir(cacheDir: String) - @JvmStatic external fun nativeSetAppId(appId: String) - @JvmStatic external fun nativeTelemetryFlush() - @JvmStatic external fun nativeTelemetryShutdown() - @JvmStatic external fun nativeInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long - @JvmStatic external fun nativeDestroy(handle: Long) - @JvmStatic external fun nativeReset(handle: Long) - @JvmStatic external fun nativeStop(handle: Long) - @JvmStatic external fun nativeComplete(handle: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String - @JvmStatic external fun nativePrefill(handle: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeTranscribe(handle: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String - @JvmStatic external fun nativeEmbed(handle: Long, text: String, normalize: Boolean): FloatArray - @JvmStatic external fun nativeRagQuery(handle: Long, query: String, topK: Int): String - @JvmStatic external fun nativeTokenize(handle: Long, text: String): IntArray - @JvmStatic external fun nativeScoreWindow(handle: Long, tokens: IntArray, start: Int, end: Int, context: Int): String - @JvmStatic external fun nativeImageEmbed(handle: Long, imagePath: String): FloatArray - @JvmStatic external fun nativeAudioEmbed(handle: Long, audioPath: String): FloatArray - @JvmStatic external fun nativeVad(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeDiarize(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeEmbedSpeaker(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray?): String - @JvmStatic external fun nativeStreamTranscribeInit(handle: Long, optionsJson: String?): Long - @JvmStatic external fun nativeStreamTranscribeProcess(handle: Long, pcmData: ByteArray): String - @JvmStatic external fun nativeStreamTranscribeStop(handle: Long): String - @JvmStatic external fun nativeIndexInit(indexDir: String, embeddingDim: Int): Long - @JvmStatic external fun nativeIndexAdd(handle: Long, ids: IntArray, documents: Array, metadatas: Array?, embeddings: Array, embeddingDim: Long): Int - @JvmStatic external fun nativeIndexDelete(handle: Long, ids: IntArray): Int - @JvmStatic external fun nativeIndexGet(handle: Long, ids: IntArray): String - @JvmStatic external fun nativeIndexQuery(handle: Long, embedding: FloatArray, topK: Long, optionsJson: String?): String - @JvmStatic external fun nativeIndexCompact(handle: Long): Int - @JvmStatic external fun nativeIndexDestroy(handle: Long) - @JvmStatic external fun nativeDetectLanguage(handle: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String - @JvmStatic external fun nativeLogSetLevel(level: Int) - @JvmStatic external fun nativeLogSetCallback(callback: CactusLogCallback?) -} - -fun cactusInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long { - val h = CactusJNI.nativeInit(modelPath, corpusDir, cacheIndex) - if (h == 0L) throw RuntimeException(CactusJNI.nativeGetLastError().ifEmpty { "Failed to initialize model" }) - return h -} -fun cactusDestroy(model: Long) = CactusJNI.nativeDestroy(model) -fun cactusReset(model: Long) = CactusJNI.nativeReset(model) -fun cactusStop(model: Long) = CactusJNI.nativeStop(model) -fun cactusGetLastError(): String = CactusJNI.nativeGetLastError() -fun cactusSetTelemetryEnvironment(cacheDir: String) = CactusJNI.nativeSetCacheDir(cacheDir) -fun cactusSetAppId(appId: String) = CactusJNI.nativeSetAppId(appId) -fun cactusTelemetryFlush() = CactusJNI.nativeTelemetryFlush() -fun cactusTelemetryShutdown() = CactusJNI.nativeTelemetryShutdown() -fun cactusComplete(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray? = null): String = - CactusJNI.nativeComplete(model, messagesJson, optionsJson, toolsJson, callback, pcmData) -fun cactusPrefill(model: Long, messagesJson: String, optionsJson: String?, toolsJson: String?, pcmData: ByteArray? = null): String = - CactusJNI.nativePrefill(model, messagesJson, optionsJson, toolsJson, pcmData) -fun cactusTranscribe(model: Long, audioPath: String?, prompt: String?, optionsJson: String?, callback: CactusTokenCallback?, pcmData: ByteArray?): String = - CactusJNI.nativeTranscribe(model, audioPath, prompt, optionsJson, callback, pcmData) -fun cactusEmbed(model: Long, text: String, normalize: Boolean): FloatArray = - CactusJNI.nativeEmbed(model, text, normalize) -fun cactusImageEmbed(model: Long, imagePath: String): FloatArray = - CactusJNI.nativeImageEmbed(model, imagePath) -fun cactusAudioEmbed(model: Long, audioPath: String): FloatArray = - CactusJNI.nativeAudioEmbed(model, audioPath) -fun cactusVad(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeVad(model, audioPath, optionsJson, pcmData) -fun cactusDiarize(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeDiarize(model, audioPath, optionsJson, pcmData) -fun cactusEmbedSpeaker(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?, maskWeights: FloatArray? = null): String = - CactusJNI.nativeEmbedSpeaker(model, audioPath, optionsJson, pcmData, maskWeights) -fun cactusRagQuery(model: Long, query: String, topK: Int): String = - CactusJNI.nativeRagQuery(model, query, topK) -fun cactusTokenize(model: Long, text: String): IntArray = - CactusJNI.nativeTokenize(model, text) -fun cactusScoreWindow(model: Long, tokens: IntArray, start: Int, end: Int, context: Int): String = - CactusJNI.nativeScoreWindow(model, tokens, start, end, context) -fun cactusStreamTranscribeStart(model: Long, optionsJson: String?): Long { - val h = CactusJNI.nativeStreamTranscribeInit(model, optionsJson) - if (h == 0L) throw RuntimeException(CactusJNI.nativeGetLastError().ifEmpty { "Failed to create stream transcriber" }) - return h -} -fun cactusStreamTranscribeProcess(stream: Long, pcmData: ByteArray): String = - CactusJNI.nativeStreamTranscribeProcess(stream, pcmData) -fun cactusStreamTranscribeStop(stream: Long): String = - CactusJNI.nativeStreamTranscribeStop(stream) -fun cactusIndexInit(indexDir: String, embeddingDim: Int): Long { - val h = CactusJNI.nativeIndexInit(indexDir, embeddingDim) - if (h == 0L) throw RuntimeException("Failed to initialize index") - return h -} -fun cactusIndexAdd(index: Long, ids: IntArray, documents: Array, embeddings: Array, metadatas: Array?): Int = - CactusJNI.nativeIndexAdd(index, ids, documents, metadatas, embeddings, embeddings[0].size.toLong()) -fun cactusIndexDelete(index: Long, ids: IntArray): Int = - CactusJNI.nativeIndexDelete(index, ids) -fun cactusIndexGet(index: Long, ids: IntArray): String = - CactusJNI.nativeIndexGet(index, ids) -fun cactusIndexQuery(index: Long, embedding: FloatArray, optionsJson: String?): String = - CactusJNI.nativeIndexQuery(index, embedding, 1000L, optionsJson) -fun cactusIndexCompact(index: Long): Int = - CactusJNI.nativeIndexCompact(index) -fun cactusIndexDestroy(index: Long) = - CactusJNI.nativeIndexDestroy(index) -fun cactusDetectLanguage(model: Long, audioPath: String?, optionsJson: String?, pcmData: ByteArray?): String = - CactusJNI.nativeDetectLanguage(model, audioPath, optionsJson, pcmData) -fun cactusLogSetLevel(level: Int) = - CactusJNI.nativeLogSetLevel(level) -fun cactusLogSetCallback(callback: CactusLogCallback?) = - CactusJNI.nativeLogSetCallback(callback) diff --git a/android/README.md b/android/README.md index 2d8051dfc..fa73fa740 100644 --- a/android/README.md +++ b/android/README.md @@ -1,6 +1,6 @@ # Android Build -Builds `libcactus` for Android (arm64-v8a). +Builds `libcactus_engine` for Android (arm64-v8a). ## Usage @@ -16,8 +16,8 @@ bash android/build.sh ## Output -- `libcactus.so` — Shared library (JNI, for Android apps) -- `libcactus.a` — Static library (for native test binaries) +- `libcactus_engine.so` — Shared library (JNI, for Android apps) +- `libcactus_engine.a` — Static library (for native test binaries) ## Options diff --git a/android/build.sh b/android/build.sh index 370a3fd83..b139e6230 100755 --- a/android/build.sh +++ b/android/build.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/bin/bash +set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" @@ -9,22 +10,22 @@ CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} BUILD_DIR="$ANDROID_DIR/build" CACTUS_CURL_ROOT=${CACTUS_CURL_ROOT:-"$PROJECT_ROOT/cactus-engine/libs/curl"} -if [ -z "$ANDROID_NDK_HOME" ]; then - if [ -n "$ANDROID_HOME" ]; then +if [ -z "${ANDROID_NDK_HOME:-}" ]; then + if [ -n "${ANDROID_HOME:-}" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) fi - if [ -z "$ANDROID_NDK_HOME" ] && [ -d "$HOME/Library/Android/sdk" ]; then + if [ -z "${ANDROID_NDK_HOME:-}" ] && [ -d "$HOME/Library/Android/sdk" ]; then ANDROID_NDK_HOME=$(ls -d "$HOME/Library/Android/sdk/ndk/"* 2>/dev/null | sort -V | tail -1) fi - if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then + if [ -z "${ANDROID_NDK_HOME:-}" ] && [ -d "/opt/homebrew/Caskroom/android-ndk" ]; then ANDROID_NDK_HOME=$(ls -d /opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK 2>/dev/null | sort -V | tail -1) fi - if [ -z "$ANDROID_NDK_HOME" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then + if [ -z "${ANDROID_NDK_HOME:-}" ] && [ -d "/opt/homebrew/share/android-ndk" ]; then ANDROID_NDK_HOME="/opt/homebrew/share/android-ndk" fi fi -if [ -z "$ANDROID_NDK_HOME" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then +if [ -z "${ANDROID_NDK_HOME:-}" ] || [ ! -d "$ANDROID_NDK_HOME" ]; then echo "Error: Android NDK not found." echo "Set ANDROID_NDK_HOME or install NDK via Android SDK Manager" exit 1 @@ -58,14 +59,14 @@ cmake -DCMAKE_TOOLCHAIN_FILE="$CMAKE_TOOLCHAIN_FILE" \ cmake --build "$BUILD_DIR" --config "$CMAKE_BUILD_TYPE" -j "$n_cpu" >/dev/null -cp "$BUILD_DIR/lib/libcactus.so" "$ANDROID_DIR/" 2>/dev/null || \ - cp "$BUILD_DIR/libcactus.so" "$ANDROID_DIR" 2>/dev/null || \ - { echo "Error: Could not find libcactus.so"; exit 1; } +cp "$BUILD_DIR/lib/libcactus_engine.so" "$ANDROID_DIR/" 2>/dev/null || \ + cp "$BUILD_DIR/libcactus_engine.so" "$ANDROID_DIR" 2>/dev/null || \ + { echo "Error: Could not find libcactus_engine.so"; exit 1; } -cp "$BUILD_DIR/lib/libcactus_static.a" "$ANDROID_DIR/libcactus.a" 2>/dev/null || \ - cp "$BUILD_DIR/libcactus_static.a" "$ANDROID_DIR/libcactus.a" 2>/dev/null || \ - { echo "Warning: Could not find libcactus_static.a"; } +cp "$BUILD_DIR/lib/libcactus_engine.a" "$ANDROID_DIR/libcactus_engine.a" 2>/dev/null || \ + cp "$BUILD_DIR/libcactus_engine.a" "$ANDROID_DIR/libcactus_engine.a" 2>/dev/null || \ + { echo "Warning: Could not find libcactus_engine.a"; } echo "Build complete!" -echo "Shared library location: $ANDROID_DIR/libcactus.so" -echo "Static library location: $ANDROID_DIR/libcactus.a" +echo "Shared library location: $ANDROID_DIR/libcactus_engine.so" +echo "Static library location: $ANDROID_DIR/libcactus_engine.a" diff --git a/android/cactus.def b/android/cactus.def deleted file mode 100644 index cf3e7ba7e..000000000 --- a/android/cactus.def +++ /dev/null @@ -1,3 +0,0 @@ -package = cactus -headers = cactus_ffi.h -headerFilter = cactus_ffi.h diff --git a/android/cactus_jni.cpp b/android/cactus_jni.cpp index 7386753f8..90a8be6d5 100644 --- a/android/cactus_jni.cpp +++ b/android/cactus_jni.cpp @@ -1,5 +1,5 @@ #include -#include "cactus.h" +#include "cactus_engine.h" struct TokenCallbackContext { JavaVM* jvm; @@ -23,12 +23,13 @@ static void token_callback_bridge(const char* token, uint32_t token_id, void* us bool attached = false; jint status = ctx->jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); if (status == JNI_EDETACHED) { - ctx->jvm->AttachCurrentThread(&env, nullptr); + if (ctx->jvm->AttachCurrentThread(&env, nullptr) != JNI_OK) return; attached = true; } - jstring jtoken = env->NewStringUTF(token); + if (!env) return; + jstring jtoken = token ? env->NewStringUTF(token) : nullptr; env->CallVoidMethod(ctx->callback, ctx->method, jtoken, static_cast(token_id)); - env->DeleteLocalRef(jtoken); + if (jtoken) env->DeleteLocalRef(jtoken); if (attached) ctx->jvm->DetachCurrentThread(); } @@ -38,14 +39,15 @@ static void log_callback_bridge(int level, const char* component, const char* me bool attached = false; jint status = ctx->jvm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); if (status == JNI_EDETACHED) { - ctx->jvm->AttachCurrentThread(&env, nullptr); + if (ctx->jvm->AttachCurrentThread(&env, nullptr) != JNI_OK) return; attached = true; } - jstring jcomponent = env->NewStringUTF(component); - jstring jmessage = env->NewStringUTF(message); + if (!env) return; + jstring jcomponent = component ? env->NewStringUTF(component) : nullptr; + jstring jmessage = message ? env->NewStringUTF(message) : nullptr; env->CallVoidMethod(ctx->callback, ctx->method, static_cast(level), jcomponent, jmessage); - env->DeleteLocalRef(jcomponent); - env->DeleteLocalRef(jmessage); + if (jcomponent) env->DeleteLocalRef(jcomponent); + if (jmessage) env->DeleteLocalRef(jmessage); if (attached) ctx->jvm->DetachCurrentThread(); } @@ -610,6 +612,11 @@ Java_com_cactus_CactusJNI_nativeLogSetCallback(JNIEnv* env, jobject, jobject cal env->GetJavaVM(&jvm); jclass cls = env->GetObjectClass(callback); jmethodID method = env->GetMethodID(cls, "onLog", "(ILjava/lang/String;Ljava/lang/String;)V"); + if (!method) { + env->ExceptionClear(); + cactus_log_set_callback(nullptr, nullptr); + return; + } g_log_callback_ctx = new LogCallbackContext{jvm, env->NewGlobalRef(callback), method}; cactus_log_set_callback(log_callback_bridge, g_log_callback_ctx); } @@ -628,6 +635,10 @@ Java_com_cactus_CactusJNI_nativeSetTelemetryEnvironment(JNIEnv* env, jobject, JNIEXPORT void JNICALL Java_com_cactus_CactusJNI_nativeSetAppId(JNIEnv* env, jobject, jstring appId) { + if (!appId) { + cactus_set_app_id(nullptr); + return; + } const char* id = env->GetStringUTFChars(appId, nullptr); cactus_set_app_id(id); env->ReleaseStringUTFChars(appId, id); diff --git a/android/mbedtls/arm64-v8a/include/.gitignore b/android/mbedtls/arm64-v8a/include/.gitignore deleted file mode 100644 index bf67d02ed..000000000 --- a/android/mbedtls/arm64-v8a/include/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -Makefile -*.sln -*.vcxproj -mbedtls/check_config diff --git a/android/mbedtls/arm64-v8a/include/CMakeLists.txt b/android/mbedtls/arm64-v8a/include/CMakeLists.txt deleted file mode 100644 index e11e2715a..000000000 --- a/android/mbedtls/arm64-v8a/include/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -option(INSTALL_MBEDTLS_HEADERS "Install Mbed TLS headers." ON) - -if(INSTALL_MBEDTLS_HEADERS) - - file(GLOB headers "mbedtls/*.h") - file(GLOB psa_headers "psa/*.h") - - install(FILES ${headers} - DESTINATION include/mbedtls - PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) - - install(FILES ${psa_headers} - DESTINATION include/psa - PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ) - -endif(INSTALL_MBEDTLS_HEADERS) - -# Make mbedtls_config.h available in an out-of-source build. ssl-opt.sh requires it. -if (ENABLE_TESTING AND NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) - link_to_source(mbedtls) - link_to_source(psa) -endif() diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/aes.h b/android/mbedtls/arm64-v8a/include/mbedtls/aes.h deleted file mode 100644 index d5eb1fd5c..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/aes.h +++ /dev/null @@ -1,631 +0,0 @@ -/** - * \file aes.h - * - * \brief This file contains AES definitions and functions. - * - * The Advanced Encryption Standard (AES) specifies a FIPS-approved - * cryptographic algorithm that can be used to protect electronic - * data. - * - * The AES algorithm is a symmetric block cipher that can - * encrypt and decrypt information. For more information, see - * FIPS Publication 197: Advanced Encryption Standard and - * ISO/IEC 18033-2:2006: Information technology -- Security - * techniques -- Encryption algorithms -- Part 2: Asymmetric - * ciphers. - * - * The AES-XTS block mode is standardized by NIST SP 800-38E - * - * and described in detail by IEEE P1619 - * . - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_AES_H -#define MBEDTLS_AES_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include -#include - -/* padlock.c and aesni.c rely on these values! */ -#define MBEDTLS_AES_ENCRYPT 1 /**< AES encryption. */ -#define MBEDTLS_AES_DECRYPT 0 /**< AES decryption. */ - -/* Error codes in range 0x0020-0x0022 */ -/** Invalid key length. */ -#define MBEDTLS_ERR_AES_INVALID_KEY_LENGTH -0x0020 -/** Invalid data input length. */ -#define MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH -0x0022 - -/* Error codes in range 0x0021-0x0025 */ -/** Invalid input data. */ -#define MBEDTLS_ERR_AES_BAD_INPUT_DATA -0x0021 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_AES_ALT) -// Regular implementation -// - -/** - * \brief The AES context-type definition. - */ -typedef struct mbedtls_aes_context { - int MBEDTLS_PRIVATE(nr); /*!< The number of rounds. */ - size_t MBEDTLS_PRIVATE(rk_offset); /*!< The offset in array elements to AES - round keys in the buffer. */ -#if defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH) && !defined(MBEDTLS_PADLOCK_C) - uint32_t MBEDTLS_PRIVATE(buf)[44]; /*!< Aligned data buffer to hold - 10 round keys for 128-bit case. */ -#else - uint32_t MBEDTLS_PRIVATE(buf)[68]; /*!< Unaligned data buffer. This buffer can - hold 32 extra Bytes, which can be used for - one of the following purposes: -
  • Alignment if VIA padlock is - used.
    • -
  • Simplifying key expansion in the 256-bit - case by generating an extra round key. -
*/ -#endif /* MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH && !MBEDTLS_PADLOCK_C */ -} -mbedtls_aes_context; - -#if defined(MBEDTLS_CIPHER_MODE_XTS) -/** - * \brief The AES XTS context-type definition. - */ -typedef struct mbedtls_aes_xts_context { - mbedtls_aes_context MBEDTLS_PRIVATE(crypt); /*!< The AES context to use for AES block - encryption or decryption. */ - mbedtls_aes_context MBEDTLS_PRIVATE(tweak); /*!< The AES context used for tweak - computation. */ -} mbedtls_aes_xts_context; -#endif /* MBEDTLS_CIPHER_MODE_XTS */ - -#else /* MBEDTLS_AES_ALT */ -#include "aes_alt.h" -#endif /* MBEDTLS_AES_ALT */ - -/** - * \brief This function initializes the specified AES context. - * - * It must be the first API called before using - * the context. - * - * \param ctx The AES context to initialize. This must not be \c NULL. - */ -void mbedtls_aes_init(mbedtls_aes_context *ctx); - -/** - * \brief This function releases and clears the specified AES context. - * - * \param ctx The AES context to clear. - * If this is \c NULL, this function does nothing. - * Otherwise, the context must have been at least initialized. - */ -void mbedtls_aes_free(mbedtls_aes_context *ctx); - -#if defined(MBEDTLS_CIPHER_MODE_XTS) -/** - * \brief This function initializes the specified AES XTS context. - * - * It must be the first API called before using - * the context. - * - * \param ctx The AES XTS context to initialize. This must not be \c NULL. - */ -void mbedtls_aes_xts_init(mbedtls_aes_xts_context *ctx); - -/** - * \brief This function releases and clears the specified AES XTS context. - * - * \param ctx The AES XTS context to clear. - * If this is \c NULL, this function does nothing. - * Otherwise, the context must have been at least initialized. - */ -void mbedtls_aes_xts_free(mbedtls_aes_xts_context *ctx); -#endif /* MBEDTLS_CIPHER_MODE_XTS */ - -/** - * \brief This function sets the encryption key. - * - * \param ctx The AES context to which the key should be bound. - * It must be initialized. - * \param key The encryption key. - * This must be a readable buffer of size \p keybits bits. - * \param keybits The size of data passed in bits. Valid options are: - *
  • 128 bits
    • - *
  • 192 bits
    • - *
  • 256 bits
- * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH on failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_setkey_enc(mbedtls_aes_context *ctx, const unsigned char *key, - unsigned int keybits); - -#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) -/** - * \brief This function sets the decryption key. - * - * \param ctx The AES context to which the key should be bound. - * It must be initialized. - * \param key The decryption key. - * This must be a readable buffer of size \p keybits bits. - * \param keybits The size of data passed. Valid options are: - *
  • 128 bits
    • - *
  • 192 bits
    • - *
  • 256 bits
- * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH on failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_setkey_dec(mbedtls_aes_context *ctx, const unsigned char *key, - unsigned int keybits); -#endif /* !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */ - -#if defined(MBEDTLS_CIPHER_MODE_XTS) -/** - * \brief This function prepares an XTS context for encryption and - * sets the encryption key. - * - * \param ctx The AES XTS context to which the key should be bound. - * It must be initialized. - * \param key The encryption key. This is comprised of the XTS key1 - * concatenated with the XTS key2. - * This must be a readable buffer of size \p keybits bits. - * \param keybits The size of \p key passed in bits. Valid options are: - *
  • 256 bits (each of key1 and key2 is a 128-bit key)
    • - *
  • 512 bits (each of key1 and key2 is a 256-bit key)
- * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH on failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_xts_setkey_enc(mbedtls_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits); - -/** - * \brief This function prepares an XTS context for decryption and - * sets the decryption key. - * - * \param ctx The AES XTS context to which the key should be bound. - * It must be initialized. - * \param key The decryption key. This is comprised of the XTS key1 - * concatenated with the XTS key2. - * This must be a readable buffer of size \p keybits bits. - * \param keybits The size of \p key passed in bits. Valid options are: - *
  • 256 bits (each of key1 and key2 is a 128-bit key)
    • - *
  • 512 bits (each of key1 and key2 is a 256-bit key)
- * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_KEY_LENGTH on failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_xts_setkey_dec(mbedtls_aes_xts_context *ctx, - const unsigned char *key, - unsigned int keybits); -#endif /* MBEDTLS_CIPHER_MODE_XTS */ - -/** - * \brief This function performs an AES single-block encryption or - * decryption operation. - * - * It performs the operation defined in the \p mode parameter - * (encrypt or decrypt), on the input data buffer defined in - * the \p input parameter. - * - * mbedtls_aes_init(), and either mbedtls_aes_setkey_enc() or - * mbedtls_aes_setkey_dec() must be called before the first - * call to this API with the same context. - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param mode The AES operation: #MBEDTLS_AES_ENCRYPT or - * #MBEDTLS_AES_DECRYPT. - * \param input The buffer holding the input data. - * It must be readable and at least \c 16 Bytes long. - * \param output The buffer where the output data will be written. - * It must be writeable and at least \c 16 Bytes long. - - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_ecb(mbedtls_aes_context *ctx, - int mode, - const unsigned char input[16], - unsigned char output[16]); - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -/** - * \brief This function performs an AES-CBC encryption or decryption operation - * on full blocks. - * - * It performs the operation defined in the \p mode - * parameter (encrypt/decrypt), on the input data buffer defined in - * the \p input parameter. - * - * It can be called as many times as needed, until all the input - * data is processed. mbedtls_aes_init(), and either - * mbedtls_aes_setkey_enc() or mbedtls_aes_setkey_dec() must be called - * before the first call to this API with the same context. - * - * \note This function operates on full blocks, that is, the input size - * must be a multiple of the AES block size of \c 16 Bytes. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the same function again on the next - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If you need to retain the contents of the IV, you should - * either save it manually or use the cipher module instead. - * - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param mode The AES operation: #MBEDTLS_AES_ENCRYPT or - * #MBEDTLS_AES_DECRYPT. - * \param length The length of the input data in Bytes. This must be a - * multiple of the block size (\c 16 Bytes). - * \param iv Initialization vector (updated after use). - * It must be a readable and writeable buffer of \c 16 Bytes. - * \param input The buffer holding the input data. - * It must be readable and of size \p length Bytes. - * \param output The buffer holding the output data. - * It must be writeable and of size \p length Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH - * on failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_cbc(mbedtls_aes_context *ctx, - int mode, - size_t length, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CBC */ - -#if defined(MBEDTLS_CIPHER_MODE_XTS) -/** - * \brief This function performs an AES-XTS encryption or decryption - * operation for an entire XTS data unit. - * - * AES-XTS encrypts or decrypts blocks based on their location as - * defined by a data unit number. The data unit number must be - * provided by \p data_unit. - * - * NIST SP 800-38E limits the maximum size of a data unit to 2^20 - * AES blocks. If the data unit is larger than this, this function - * returns #MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH. - * - * \param ctx The AES XTS context to use for AES XTS operations. - * It must be initialized and bound to a key. - * \param mode The AES operation: #MBEDTLS_AES_ENCRYPT or - * #MBEDTLS_AES_DECRYPT. - * \param length The length of a data unit in Bytes. This can be any - * length between 16 bytes and 2^24 bytes inclusive - * (between 1 and 2^20 block cipher blocks). - * \param data_unit The address of the data unit encoded as an array of 16 - * bytes in little-endian format. For disk encryption, this - * is typically the index of the block device sector that - * contains the data. - * \param input The buffer holding the input data (which is an entire - * data unit). This function reads \p length Bytes from \p - * input. - * \param output The buffer holding the output data (which is an entire - * data unit). This function writes \p length Bytes to \p - * output. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_AES_INVALID_INPUT_LENGTH if \p length is - * smaller than an AES block in size (16 Bytes) or if \p - * length is larger than 2^20 blocks (16 MiB). - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_xts(mbedtls_aes_xts_context *ctx, - int mode, - size_t length, - const unsigned char data_unit[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_XTS */ - -#if defined(MBEDTLS_CIPHER_MODE_CFB) -/** - * \brief This function performs an AES-CFB128 encryption or decryption - * operation. - * - * It performs the operation defined in the \p mode - * parameter (encrypt or decrypt), on the input data buffer - * defined in the \p input parameter. - * - * For CFB, you must set up the context with mbedtls_aes_setkey_enc(), - * regardless of whether you are performing an encryption or decryption - * operation, that is, regardless of the \p mode parameter. This is - * because CFB mode uses the same key schedule for encryption and - * decryption. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the same function again on the next - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If you need to retain the contents of the - * IV, you must either save it manually or use the cipher - * module instead. - * - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param mode The AES operation: #MBEDTLS_AES_ENCRYPT or - * #MBEDTLS_AES_DECRYPT. - * \param length The length of the input data in Bytes. - * \param iv_off The offset in IV (updated after use). - * It must point to a valid \c size_t. - * \param iv The initialization vector (updated after use). - * It must be a readable and writeable buffer of \c 16 Bytes. - * \param input The buffer holding the input data. - * It must be readable and of size \p length Bytes. - * \param output The buffer holding the output data. - * It must be writeable and of size \p length Bytes. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_cfb128(mbedtls_aes_context *ctx, - int mode, - size_t length, - size_t *iv_off, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function performs an AES-CFB8 encryption or decryption - * operation. - * - * It performs the operation defined in the \p mode - * parameter (encrypt/decrypt), on the input data buffer defined - * in the \p input parameter. - * - * Due to the nature of CFB, you must use the same key schedule for - * both encryption and decryption operations. Therefore, you must - * use the context initialized with mbedtls_aes_setkey_enc() for - * both #MBEDTLS_AES_ENCRYPT and #MBEDTLS_AES_DECRYPT. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the same function again on the next - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If you need to retain the contents of the - * IV, you should either save it manually or use the cipher - * module instead. - * - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param mode The AES operation: #MBEDTLS_AES_ENCRYPT or - * #MBEDTLS_AES_DECRYPT - * \param length The length of the input data. - * \param iv The initialization vector (updated after use). - * It must be a readable and writeable buffer of \c 16 Bytes. - * \param input The buffer holding the input data. - * It must be readable and of size \p length Bytes. - * \param output The buffer holding the output data. - * It must be writeable and of size \p length Bytes. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_cfb8(mbedtls_aes_context *ctx, - int mode, - size_t length, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); -#endif /*MBEDTLS_CIPHER_MODE_CFB */ - -#if defined(MBEDTLS_CIPHER_MODE_OFB) -/** - * \brief This function performs an AES-OFB (Output Feedback Mode) - * encryption or decryption operation. - * - * For OFB, you must set up the context with - * mbedtls_aes_setkey_enc(), regardless of whether you are - * performing an encryption or decryption operation. This is - * because OFB mode uses the same key schedule for encryption and - * decryption. - * - * The OFB operation is identical for encryption or decryption, - * therefore no operation mode needs to be specified. - * - * \note Upon exit, the content of iv, the Initialisation Vector, is - * updated so that you can call the same function again on the next - * block(s) of data and get the same result as if it was encrypted - * in one call. This allows a "streaming" usage, by initialising - * iv_off to 0 before the first call, and preserving its value - * between calls. - * - * For non-streaming use, the iv should be initialised on each call - * to a unique value, and iv_off set to 0 on each call. - * - * If you need to retain the contents of the initialisation vector, - * you must either save it manually or use the cipher module - * instead. - * - * \warning For the OFB mode, the initialisation vector must be unique - * every encryption operation. Reuse of an initialisation vector - * will compromise security. - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param length The length of the input data. - * \param iv_off The offset in IV (updated after use). - * It must point to a valid \c size_t. - * \param iv The initialization vector (updated after use). - * It must be a readable and writeable buffer of \c 16 Bytes. - * \param input The buffer holding the input data. - * It must be readable and of size \p length Bytes. - * \param output The buffer holding the output data. - * It must be writeable and of size \p length Bytes. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_ofb(mbedtls_aes_context *ctx, - size_t length, - size_t *iv_off, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); - -#endif /* MBEDTLS_CIPHER_MODE_OFB */ - -#if defined(MBEDTLS_CIPHER_MODE_CTR) -/** - * \brief This function performs an AES-CTR encryption or decryption - * operation. - * - * Due to the nature of CTR, you must use the same key schedule - * for both encryption and decryption operations. Therefore, you - * must use the context initialized with mbedtls_aes_setkey_enc() - * for both #MBEDTLS_AES_ENCRYPT and #MBEDTLS_AES_DECRYPT. - * - * \warning You must never reuse a nonce value with the same key. Doing so - * would void the encryption for the two messages encrypted with - * the same nonce and key. - * - * There are two common strategies for managing nonces with CTR: - * - * 1. You can handle everything as a single message processed over - * successive calls to this function. In that case, you want to - * set \p nonce_counter and \p nc_off to 0 for the first call, and - * then preserve the values of \p nonce_counter, \p nc_off and \p - * stream_block across calls to this function as they will be - * updated by this function. - * - * With this strategy, you must not encrypt more than 2**128 - * blocks of data with the same key. - * - * 2. You can encrypt separate messages by dividing the \p - * nonce_counter buffer in two areas: the first one used for a - * per-message nonce, handled by yourself, and the second one - * updated by this function internally. - * - * For example, you might reserve the first 12 bytes for the - * per-message nonce, and the last 4 bytes for internal use. In that - * case, before calling this function on a new message you need to - * set the first 12 bytes of \p nonce_counter to your chosen nonce - * value, the last 4 to 0, and \p nc_off to 0 (which will cause \p - * stream_block to be ignored). That way, you can encrypt at most - * 2**96 messages of up to 2**32 blocks each with the same key. - * - * The per-message nonce (or information sufficient to reconstruct - * it) needs to be communicated with the ciphertext and must be unique. - * The recommended way to ensure uniqueness is to use a message - * counter. An alternative is to generate random nonces, but this - * limits the number of messages that can be securely encrypted: - * for example, with 96-bit random nonces, you should not encrypt - * more than 2**32 messages with the same key. - * - * Note that for both strategies, sizes are measured in blocks and - * that an AES block is 16 bytes. - * - * \warning Upon return, \p stream_block contains sensitive data. Its - * content must not be written to insecure storage and should be - * securely discarded as soon as it's no longer needed. - * - * \param ctx The AES context to use for encryption or decryption. - * It must be initialized and bound to a key. - * \param length The length of the input data. - * \param nc_off The offset in the current \p stream_block, for - * resuming within the current cipher stream. The - * offset pointer should be 0 at the start of a stream. - * It must point to a valid \c size_t. - * \param nonce_counter The 128-bit nonce and counter. - * It must be a readable-writeable buffer of \c 16 Bytes. - * \param stream_block The saved stream block for resuming. This is - * overwritten by the function. - * It must be a readable-writeable buffer of \c 16 Bytes. - * \param input The buffer holding the input data. - * It must be readable and of size \p length Bytes. - * \param output The buffer holding the output data. - * It must be writeable and of size \p length Bytes. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_aes_crypt_ctr(mbedtls_aes_context *ctx, - size_t length, - size_t *nc_off, - unsigned char nonce_counter[16], - unsigned char stream_block[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CTR */ - -/** - * \brief Internal AES block encryption function. This is only - * exposed to allow overriding it using - * \c MBEDTLS_AES_ENCRYPT_ALT. - * - * \param ctx The AES context to use for encryption. - * \param input The plaintext block. - * \param output The output (ciphertext) block. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_internal_aes_encrypt(mbedtls_aes_context *ctx, - const unsigned char input[16], - unsigned char output[16]); - -#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) -/** - * \brief Internal AES block decryption function. This is only - * exposed to allow overriding it using see - * \c MBEDTLS_AES_DECRYPT_ALT. - * - * \param ctx The AES context to use for decryption. - * \param input The ciphertext block. - * \param output The output (plaintext) block. - * - * \return \c 0 on success. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_internal_aes_decrypt(mbedtls_aes_context *ctx, - const unsigned char input[16], - unsigned char output[16]); -#endif /* !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */ - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -MBEDTLS_CHECK_RETURN_CRITICAL -int mbedtls_aes_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* aes.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/aria.h b/android/mbedtls/arm64-v8a/include/mbedtls/aria.h deleted file mode 100644 index c685fc314..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/aria.h +++ /dev/null @@ -1,343 +0,0 @@ -/** - * \file aria.h - * - * \brief ARIA block cipher - * - * The ARIA algorithm is a symmetric block cipher that can encrypt and - * decrypt information. It is defined by the Korean Agency for - * Technology and Standards (KATS) in KS X 1213:2004 (in - * Korean, but see http://210.104.33.10/ARIA/index-e.html in English) - * and also described by the IETF in RFC 5794. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_ARIA_H -#define MBEDTLS_ARIA_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -#include "mbedtls/platform_util.h" - -#define MBEDTLS_ARIA_ENCRYPT 1 /**< ARIA encryption. */ -#define MBEDTLS_ARIA_DECRYPT 0 /**< ARIA decryption. */ - -#define MBEDTLS_ARIA_BLOCKSIZE 16 /**< ARIA block size in bytes. */ -#define MBEDTLS_ARIA_MAX_ROUNDS 16 /**< Maximum number of rounds in ARIA. */ -#define MBEDTLS_ARIA_MAX_KEYSIZE 32 /**< Maximum size of an ARIA key in bytes. */ - -/** Bad input data. */ -#define MBEDTLS_ERR_ARIA_BAD_INPUT_DATA -0x005C - -/** Invalid data input length. */ -#define MBEDTLS_ERR_ARIA_INVALID_INPUT_LENGTH -0x005E - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_ARIA_ALT) -// Regular implementation -// - -/** - * \brief The ARIA context-type definition. - */ -typedef struct mbedtls_aria_context { - unsigned char MBEDTLS_PRIVATE(nr); /*!< The number of rounds (12, 14 or 16) */ - /*! The ARIA round keys. */ - uint32_t MBEDTLS_PRIVATE(rk)[MBEDTLS_ARIA_MAX_ROUNDS + 1][MBEDTLS_ARIA_BLOCKSIZE / 4]; -} -mbedtls_aria_context; - -#else /* MBEDTLS_ARIA_ALT */ -#include "aria_alt.h" -#endif /* MBEDTLS_ARIA_ALT */ - -/** - * \brief This function initializes the specified ARIA context. - * - * It must be the first API called before using - * the context. - * - * \param ctx The ARIA context to initialize. This must not be \c NULL. - */ -void mbedtls_aria_init(mbedtls_aria_context *ctx); - -/** - * \brief This function releases and clears the specified ARIA context. - * - * \param ctx The ARIA context to clear. This may be \c NULL, in which - * case this function returns immediately. If it is not \c NULL, - * it must point to an initialized ARIA context. - */ -void mbedtls_aria_free(mbedtls_aria_context *ctx); - -/** - * \brief This function sets the encryption key. - * - * \param ctx The ARIA context to which the key should be bound. - * This must be initialized. - * \param key The encryption key. This must be a readable buffer - * of size \p keybits Bits. - * \param keybits The size of \p key in Bits. Valid options are: - *
  • 128 bits
    • - *
  • 192 bits
    • - *
  • 256 bits
- * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_setkey_enc(mbedtls_aria_context *ctx, - const unsigned char *key, - unsigned int keybits); - -#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) -/** - * \brief This function sets the decryption key. - * - * \param ctx The ARIA context to which the key should be bound. - * This must be initialized. - * \param key The decryption key. This must be a readable buffer - * of size \p keybits Bits. - * \param keybits The size of data passed. Valid options are: - *
  • 128 bits
    • - *
  • 192 bits
    • - *
  • 256 bits
- * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_setkey_dec(mbedtls_aria_context *ctx, - const unsigned char *key, - unsigned int keybits); -#endif /* !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */ - -/** - * \brief This function performs an ARIA single-block encryption or - * decryption operation. - * - * It performs encryption or decryption (depending on whether - * the key was set for encryption on decryption) on the input - * data buffer defined in the \p input parameter. - * - * mbedtls_aria_init(), and either mbedtls_aria_setkey_enc() or - * mbedtls_aria_setkey_dec() must be called before the first - * call to this API with the same context. - * - * \param ctx The ARIA context to use for encryption or decryption. - * This must be initialized and bound to a key. - * \param input The 16-Byte buffer holding the input data. - * \param output The 16-Byte buffer holding the output data. - - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_crypt_ecb(mbedtls_aria_context *ctx, - const unsigned char input[MBEDTLS_ARIA_BLOCKSIZE], - unsigned char output[MBEDTLS_ARIA_BLOCKSIZE]); - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -/** - * \brief This function performs an ARIA-CBC encryption or decryption operation - * on full blocks. - * - * It performs the operation defined in the \p mode - * parameter (encrypt/decrypt), on the input data buffer defined in - * the \p input parameter. - * - * It can be called as many times as needed, until all the input - * data is processed. mbedtls_aria_init(), and either - * mbedtls_aria_setkey_enc() or mbedtls_aria_setkey_dec() must be called - * before the first call to this API with the same context. - * - * \note This function operates on aligned blocks, that is, the input size - * must be a multiple of the ARIA block size of 16 Bytes. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the same function again on the next - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If you need to retain the contents of the IV, you should - * either save it manually or use the cipher module instead. - * - * - * \param ctx The ARIA context to use for encryption or decryption. - * This must be initialized and bound to a key. - * \param mode The mode of operation. This must be either - * #MBEDTLS_ARIA_ENCRYPT for encryption, or - * #MBEDTLS_ARIA_DECRYPT for decryption. - * \param length The length of the input data in Bytes. This must be a - * multiple of the block size (16 Bytes). - * \param iv Initialization vector (updated after use). - * This must be a readable buffer of size 16 Bytes. - * \param input The buffer holding the input data. This must - * be a readable buffer of length \p length Bytes. - * \param output The buffer holding the output data. This must - * be a writable buffer of length \p length Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_crypt_cbc(mbedtls_aria_context *ctx, - int mode, - size_t length, - unsigned char iv[MBEDTLS_ARIA_BLOCKSIZE], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CBC */ - -#if defined(MBEDTLS_CIPHER_MODE_CFB) -/** - * \brief This function performs an ARIA-CFB128 encryption or decryption - * operation. - * - * It performs the operation defined in the \p mode - * parameter (encrypt or decrypt), on the input data buffer - * defined in the \p input parameter. - * - * For CFB, you must set up the context with mbedtls_aria_setkey_enc(), - * regardless of whether you are performing an encryption or decryption - * operation, that is, regardless of the \p mode parameter. This is - * because CFB mode uses the same key schedule for encryption and - * decryption. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the same function again on the next - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If you need to retain the contents of the - * IV, you must either save it manually or use the cipher - * module instead. - * - * - * \param ctx The ARIA context to use for encryption or decryption. - * This must be initialized and bound to a key. - * \param mode The mode of operation. This must be either - * #MBEDTLS_ARIA_ENCRYPT for encryption, or - * #MBEDTLS_ARIA_DECRYPT for decryption. - * \param length The length of the input data \p input in Bytes. - * \param iv_off The offset in IV (updated after use). - * This must not be larger than 15. - * \param iv The initialization vector (updated after use). - * This must be a readable buffer of size 16 Bytes. - * \param input The buffer holding the input data. This must - * be a readable buffer of length \p length Bytes. - * \param output The buffer holding the output data. This must - * be a writable buffer of length \p length Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_crypt_cfb128(mbedtls_aria_context *ctx, - int mode, - size_t length, - size_t *iv_off, - unsigned char iv[MBEDTLS_ARIA_BLOCKSIZE], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CFB */ - -#if defined(MBEDTLS_CIPHER_MODE_CTR) -/** - * \brief This function performs an ARIA-CTR encryption or decryption - * operation. - * - * Due to the nature of CTR, you must use the same key schedule - * for both encryption and decryption operations. Therefore, you - * must use the context initialized with mbedtls_aria_setkey_enc() - * for both #MBEDTLS_ARIA_ENCRYPT and #MBEDTLS_ARIA_DECRYPT. - * - * \warning You must never reuse a nonce value with the same key. Doing so - * would void the encryption for the two messages encrypted with - * the same nonce and key. - * - * There are two common strategies for managing nonces with CTR: - * - * 1. You can handle everything as a single message processed over - * successive calls to this function. In that case, you want to - * set \p nonce_counter and \p nc_off to 0 for the first call, and - * then preserve the values of \p nonce_counter, \p nc_off and \p - * stream_block across calls to this function as they will be - * updated by this function. - * - * With this strategy, you must not encrypt more than 2**128 - * blocks of data with the same key. - * - * 2. You can encrypt separate messages by dividing the \p - * nonce_counter buffer in two areas: the first one used for a - * per-message nonce, handled by yourself, and the second one - * updated by this function internally. - * - * For example, you might reserve the first 12 bytes for the - * per-message nonce, and the last 4 bytes for internal use. In that - * case, before calling this function on a new message you need to - * set the first 12 bytes of \p nonce_counter to your chosen nonce - * value, the last 4 to 0, and \p nc_off to 0 (which will cause \p - * stream_block to be ignored). That way, you can encrypt at most - * 2**96 messages of up to 2**32 blocks each with the same key. - * - * The per-message nonce (or information sufficient to reconstruct - * it) needs to be communicated with the ciphertext and must be unique. - * The recommended way to ensure uniqueness is to use a message - * counter. An alternative is to generate random nonces, but this - * limits the number of messages that can be securely encrypted: - * for example, with 96-bit random nonces, you should not encrypt - * more than 2**32 messages with the same key. - * - * Note that for both strategies, sizes are measured in blocks and - * that an ARIA block is 16 bytes. - * - * \warning Upon return, \p stream_block contains sensitive data. Its - * content must not be written to insecure storage and should be - * securely discarded as soon as it's no longer needed. - * - * \param ctx The ARIA context to use for encryption or decryption. - * This must be initialized and bound to a key. - * \param length The length of the input data \p input in Bytes. - * \param nc_off The offset in Bytes in the current \p stream_block, - * for resuming within the current cipher stream. The - * offset pointer should be \c 0 at the start of a - * stream. This must not be larger than \c 15 Bytes. - * \param nonce_counter The 128-bit nonce and counter. This must point to - * a read/write buffer of length \c 16 bytes. - * \param stream_block The saved stream block for resuming. This must - * point to a read/write buffer of length \c 16 bytes. - * This is overwritten by the function. - * \param input The buffer holding the input data. This must - * be a readable buffer of length \p length Bytes. - * \param output The buffer holding the output data. This must - * be a writable buffer of length \p length Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_aria_crypt_ctr(mbedtls_aria_context *ctx, - size_t length, - size_t *nc_off, - unsigned char nonce_counter[MBEDTLS_ARIA_BLOCKSIZE], - unsigned char stream_block[MBEDTLS_ARIA_BLOCKSIZE], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CTR */ - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine. - * - * \return \c 0 on success, or \c 1 on failure. - */ -int mbedtls_aria_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* aria.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/asn1.h b/android/mbedtls/arm64-v8a/include/mbedtls/asn1.h deleted file mode 100644 index ff019f432..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/asn1.h +++ /dev/null @@ -1,642 +0,0 @@ -/** - * \file asn1.h - * - * \brief Generic ASN.1 parsing - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_ASN1_H -#define MBEDTLS_ASN1_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include - -#if defined(MBEDTLS_BIGNUM_C) -#include "mbedtls/bignum.h" -#endif - -/** - * \addtogroup asn1_module - * \{ - */ - -/** - * \name ASN1 Error codes - * These error codes are combined with other error codes for - * higher error granularity. - * e.g. X.509 and PKCS #7 error codes - * ASN1 is a standard to specify data structures. - * \{ - */ -/** Out of data when parsing an ASN1 data structure. */ -#define MBEDTLS_ERR_ASN1_OUT_OF_DATA -0x0060 -/** ASN1 tag was of an unexpected value. */ -#define MBEDTLS_ERR_ASN1_UNEXPECTED_TAG -0x0062 -/** Error when trying to determine the length or invalid length. */ -#define MBEDTLS_ERR_ASN1_INVALID_LENGTH -0x0064 -/** Actual length differs from expected length. */ -#define MBEDTLS_ERR_ASN1_LENGTH_MISMATCH -0x0066 -/** Data is invalid. */ -#define MBEDTLS_ERR_ASN1_INVALID_DATA -0x0068 -/** Memory allocation failed */ -#define MBEDTLS_ERR_ASN1_ALLOC_FAILED -0x006A -/** Buffer too small when writing ASN.1 data structure. */ -#define MBEDTLS_ERR_ASN1_BUF_TOO_SMALL -0x006C - -/** \} name ASN1 Error codes */ - -/** - * \name DER constants - * These constants comply with the DER encoded ASN.1 type tags. - * DER encoding uses hexadecimal representation. - * An example DER sequence is:\n - * - 0x02 -- tag indicating INTEGER - * - 0x01 -- length in octets - * - 0x05 -- value - * Such sequences are typically read into \c ::mbedtls_x509_buf. - * \{ - */ -#define MBEDTLS_ASN1_BOOLEAN 0x01 -#define MBEDTLS_ASN1_INTEGER 0x02 -#define MBEDTLS_ASN1_BIT_STRING 0x03 -#define MBEDTLS_ASN1_OCTET_STRING 0x04 -#define MBEDTLS_ASN1_NULL 0x05 -#define MBEDTLS_ASN1_OID 0x06 -#define MBEDTLS_ASN1_ENUMERATED 0x0A -#define MBEDTLS_ASN1_UTF8_STRING 0x0C -#define MBEDTLS_ASN1_SEQUENCE 0x10 -#define MBEDTLS_ASN1_SET 0x11 -#define MBEDTLS_ASN1_PRINTABLE_STRING 0x13 -#define MBEDTLS_ASN1_T61_STRING 0x14 -#define MBEDTLS_ASN1_IA5_STRING 0x16 -#define MBEDTLS_ASN1_UTC_TIME 0x17 -#define MBEDTLS_ASN1_GENERALIZED_TIME 0x18 -#define MBEDTLS_ASN1_UNIVERSAL_STRING 0x1C -#define MBEDTLS_ASN1_BMP_STRING 0x1E -#define MBEDTLS_ASN1_PRIMITIVE 0x00 -#define MBEDTLS_ASN1_CONSTRUCTED 0x20 -#define MBEDTLS_ASN1_CONTEXT_SPECIFIC 0x80 - -/* Slightly smaller way to check if tag is a string tag - * compared to canonical implementation. */ -#define MBEDTLS_ASN1_IS_STRING_TAG(tag) \ - ((unsigned int) (tag) < 32u && ( \ - ((1u << (tag)) & ((1u << MBEDTLS_ASN1_BMP_STRING) | \ - (1u << MBEDTLS_ASN1_UTF8_STRING) | \ - (1u << MBEDTLS_ASN1_T61_STRING) | \ - (1u << MBEDTLS_ASN1_IA5_STRING) | \ - (1u << MBEDTLS_ASN1_UNIVERSAL_STRING) | \ - (1u << MBEDTLS_ASN1_PRINTABLE_STRING))) != 0)) - -/* - * Bit masks for each of the components of an ASN.1 tag as specified in - * ITU X.690 (08/2015), section 8.1 "General rules for encoding", - * paragraph 8.1.2.2: - * - * Bit 8 7 6 5 1 - * +-------+-----+------------+ - * | Class | P/C | Tag number | - * +-------+-----+------------+ - */ -#define MBEDTLS_ASN1_TAG_CLASS_MASK 0xC0 -#define MBEDTLS_ASN1_TAG_PC_MASK 0x20 -#define MBEDTLS_ASN1_TAG_VALUE_MASK 0x1F - -/** \} name DER constants */ - -/** Returns the size of the binary string, without the trailing \\0 */ -#define MBEDTLS_OID_SIZE(x) (sizeof(x) - 1) - -/** - * Compares an mbedtls_asn1_buf structure to a reference OID. - * - * Only works for 'defined' oid_str values (MBEDTLS_OID_HMAC_SHA1), you cannot use a - * 'unsigned char *oid' here! - */ -#define MBEDTLS_OID_CMP(oid_str, oid_buf) \ - ((MBEDTLS_OID_SIZE(oid_str) != (oid_buf)->len) || \ - memcmp((oid_str), (oid_buf)->p, (oid_buf)->len) != 0) - -#define MBEDTLS_OID_CMP_RAW(oid_str, oid_buf, oid_buf_len) \ - ((MBEDTLS_OID_SIZE(oid_str) != (oid_buf_len)) || \ - memcmp((oid_str), (oid_buf), (oid_buf_len)) != 0) - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \name Functions to parse ASN.1 data structures - * \{ - */ - -/** - * Type-length-value structure that allows for ASN1 using DER. - */ -typedef struct mbedtls_asn1_buf { - int tag; /**< ASN1 type, e.g. MBEDTLS_ASN1_UTF8_STRING. */ - size_t len; /**< ASN1 length, in octets. */ - unsigned char *p; /**< ASN1 data, e.g. in ASCII. */ -} -mbedtls_asn1_buf; - -/** - * Container for ASN1 bit strings. - */ -typedef struct mbedtls_asn1_bitstring { - size_t len; /**< ASN1 length, in octets. */ - unsigned char unused_bits; /**< Number of unused bits at the end of the string */ - unsigned char *p; /**< Raw ASN1 data for the bit string */ -} -mbedtls_asn1_bitstring; - -/** - * Container for a sequence of ASN.1 items - */ -typedef struct mbedtls_asn1_sequence { - mbedtls_asn1_buf buf; /**< Buffer containing the given ASN.1 item. */ - - /** The next entry in the sequence. - * - * The details of memory management for sequences are not documented and - * may change in future versions. Set this field to \p NULL when - * initializing a structure, and do not modify it except via Mbed TLS - * library functions. - */ - struct mbedtls_asn1_sequence *next; -} -mbedtls_asn1_sequence; - -/** - * Container for a sequence or list of 'named' ASN.1 data items - */ -typedef struct mbedtls_asn1_named_data { - mbedtls_asn1_buf oid; /**< The object identifier. */ - mbedtls_asn1_buf val; /**< The named value. */ - - /** The next entry in the sequence. - * - * The details of memory management for named data sequences are not - * documented and may change in future versions. Set this field to \p NULL - * when initializing a structure, and do not modify it except via Mbed TLS - * library functions. - */ - struct mbedtls_asn1_named_data *next; - - /** Merge next item into the current one? - * - * This field exists for the sake of Mbed TLS's X.509 certificate parsing - * code and may change in future versions of the library. - */ - unsigned char MBEDTLS_PRIVATE(next_merged); -} -mbedtls_asn1_named_data; - -#if defined(MBEDTLS_ASN1_PARSE_C) || defined(MBEDTLS_X509_CREATE_C) || \ - defined(MBEDTLS_PSA_UTIL_HAVE_ECDSA) -/** - * \brief Get the length of an ASN.1 element. - * Updates the pointer to immediately behind the length. - * - * \param p On entry, \c *p points to the first byte of the length, - * i.e. immediately after the tag. - * On successful completion, \c *p points to the first byte - * after the length, i.e. the first byte of the content. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param len On successful completion, \c *len contains the length - * read from the ASN.1 input. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_OUT_OF_DATA if the ASN.1 element - * would end beyond \p end. - * \return #MBEDTLS_ERR_ASN1_INVALID_LENGTH if the length is unparsable. - */ -int mbedtls_asn1_get_len(unsigned char **p, - const unsigned char *end, - size_t *len); - -/** - * \brief Get the tag and length of the element. - * Check for the requested tag. - * Updates the pointer to immediately behind the tag and length. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * after the length, i.e. the first byte of the content. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param len On successful completion, \c *len contains the length - * read from the ASN.1 input. - * \param tag The expected tag. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_UNEXPECTED_TAG if the data does not start - * with the requested tag. - * \return #MBEDTLS_ERR_ASN1_OUT_OF_DATA if the ASN.1 element - * would end beyond \p end. - * \return #MBEDTLS_ERR_ASN1_INVALID_LENGTH if the length is unparsable. - */ -int mbedtls_asn1_get_tag(unsigned char **p, - const unsigned char *end, - size_t *len, int tag); -#endif /* MBEDTLS_ASN1_PARSE_C || MBEDTLS_X509_CREATE_C || MBEDTLS_PSA_UTIL_HAVE_ECDSA */ - -#if defined(MBEDTLS_ASN1_PARSE_C) -/** - * \brief Retrieve a boolean ASN.1 tag and its value. - * Updates the pointer to immediately behind the full tag. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the ASN.1 element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param val On success, the parsed value (\c 0 or \c 1). - * - * \return 0 if successful. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 BOOLEAN. - */ -int mbedtls_asn1_get_bool(unsigned char **p, - const unsigned char *end, - int *val); - -/** - * \brief Retrieve an integer ASN.1 tag and its value. - * Updates the pointer to immediately behind the full tag. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the ASN.1 element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param val On success, the parsed value. - * - * \return 0 if successful. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 INTEGER. - * \return #MBEDTLS_ERR_ASN1_INVALID_LENGTH if the parsed value does - * not fit in an \c int. - */ -int mbedtls_asn1_get_int(unsigned char **p, - const unsigned char *end, - int *val); - -/** - * \brief Retrieve an enumerated ASN.1 tag and its value. - * Updates the pointer to immediately behind the full tag. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the ASN.1 element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param val On success, the parsed value. - * - * \return 0 if successful. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 ENUMERATED. - * \return #MBEDTLS_ERR_ASN1_INVALID_LENGTH if the parsed value does - * not fit in an \c int. - */ -int mbedtls_asn1_get_enum(unsigned char **p, - const unsigned char *end, - int *val); - -/** - * \brief Retrieve a bitstring ASN.1 tag and its value. - * Updates the pointer to immediately behind the full tag. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p is equal to \p end. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param bs On success, ::mbedtls_asn1_bitstring information about - * the parsed value. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_LENGTH_MISMATCH if the input contains - * extra data after a valid BIT STRING. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 BIT STRING. - */ -int mbedtls_asn1_get_bitstring(unsigned char **p, const unsigned char *end, - mbedtls_asn1_bitstring *bs); - -/** - * \brief Retrieve a bitstring ASN.1 tag without unused bits and its - * value. - * Updates the pointer to the beginning of the bit/octet string. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * of the content of the BIT STRING. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param len On success, \c *len is the length of the content in bytes. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_INVALID_DATA if the input starts with - * a valid BIT STRING with a nonzero number of unused bits. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 BIT STRING. - */ -int mbedtls_asn1_get_bitstring_null(unsigned char **p, - const unsigned char *end, - size_t *len); - -/** - * \brief Parses and splits an ASN.1 "SEQUENCE OF ". - * Updates the pointer to immediately behind the full sequence tag. - * - * This function allocates memory for the sequence elements. You can free - * the allocated memory with mbedtls_asn1_sequence_free(). - * - * \note On error, this function may return a partial list in \p cur. - * You must set `cur->next = NULL` before calling this function! - * Otherwise it is impossible to distinguish a previously non-null - * pointer from a pointer to an object allocated by this function. - * - * \note If the sequence is empty, this function does not modify - * \c *cur. If the sequence is valid and non-empty, this - * function sets `cur->buf.tag` to \p tag. This allows - * callers to distinguish between an empty sequence and - * a one-element sequence. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p is equal to \p end. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param cur A ::mbedtls_asn1_sequence which this function fills. - * When this function returns, \c *cur is the head of a linked - * list. Each node in this list is allocated with - * mbedtls_calloc() apart from \p cur itself, and should - * therefore be freed with mbedtls_free(). - * The list describes the content of the sequence. - * The head of the list (i.e. \c *cur itself) describes the - * first element, `*cur->next` describes the second element, etc. - * For each element, `buf.tag == tag`, `buf.len` is the length - * of the content of the content of the element, and `buf.p` - * points to the first byte of the content (i.e. immediately - * past the length of the element). - * Note that list elements may be allocated even on error. - * \param tag Each element of the sequence must have this tag. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_LENGTH_MISMATCH if the input contains - * extra data after a valid SEQUENCE OF \p tag. - * \return #MBEDTLS_ERR_ASN1_UNEXPECTED_TAG if the input starts with - * an ASN.1 SEQUENCE in which an element has a tag that - * is different from \p tag. - * \return #MBEDTLS_ERR_ASN1_ALLOC_FAILED if a memory allocation failed. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 SEQUENCE. - */ -int mbedtls_asn1_get_sequence_of(unsigned char **p, - const unsigned char *end, - mbedtls_asn1_sequence *cur, - int tag); -/** - * \brief Free a heap-allocated linked list presentation of - * an ASN.1 sequence, including the first element. - * - * There are two common ways to manage the memory used for the representation - * of a parsed ASN.1 sequence: - * - Allocate a head node `mbedtls_asn1_sequence *head` with mbedtls_calloc(). - * Pass this node as the `cur` argument to mbedtls_asn1_get_sequence_of(). - * When you have finished processing the sequence, - * call mbedtls_asn1_sequence_free() on `head`. - * - Allocate a head node `mbedtls_asn1_sequence *head` in any manner, - * for example on the stack. Make sure that `head->next == NULL`. - * Pass `head` as the `cur` argument to mbedtls_asn1_get_sequence_of(). - * When you have finished processing the sequence, - * call mbedtls_asn1_sequence_free() on `head->cur`, - * then free `head` itself in the appropriate manner. - * - * \param seq The address of the first sequence component. This may - * be \c NULL, in which case this functions returns - * immediately. - */ -void mbedtls_asn1_sequence_free(mbedtls_asn1_sequence *seq); - -/** - * \brief Traverse an ASN.1 SEQUENCE container and - * call a callback for each entry. - * - * This function checks that the input is a SEQUENCE of elements that - * each have a "must" tag, and calls a callback function on the elements - * that have a "may" tag. - * - * For example, to validate that the input is a SEQUENCE of `tag1` and call - * `cb` on each element, use - * ``` - * mbedtls_asn1_traverse_sequence_of(&p, end, 0xff, tag1, 0, 0, cb, ctx); - * ``` - * - * To validate that the input is a SEQUENCE of ANY and call `cb` on - * each element, use - * ``` - * mbedtls_asn1_traverse_sequence_of(&p, end, 0, 0, 0, 0, cb, ctx); - * ``` - * - * To validate that the input is a SEQUENCE of CHOICE {NULL, OCTET STRING} - * and call `cb` on each element that is an OCTET STRING, use - * ``` - * mbedtls_asn1_traverse_sequence_of(&p, end, 0xfe, 0x04, 0xff, 0x04, cb, ctx); - * ``` - * - * The callback is called on the elements with a "may" tag from left to - * right. If the input is not a valid SEQUENCE of elements with a "must" tag, - * the callback is called on the elements up to the leftmost point where - * the input is invalid. - * - * \warning This function is still experimental and may change - * at any time. - * - * \param p The address of the pointer to the beginning of - * the ASN.1 SEQUENCE header. This is updated to - * point to the end of the ASN.1 SEQUENCE container - * on a successful invocation. - * \param end The end of the ASN.1 SEQUENCE container. - * \param tag_must_mask A mask to be applied to the ASN.1 tags found within - * the SEQUENCE before comparing to \p tag_must_val. - * \param tag_must_val The required value of each ASN.1 tag found in the - * SEQUENCE, after masking with \p tag_must_mask. - * Mismatching tags lead to an error. - * For example, a value of \c 0 for both \p tag_must_mask - * and \p tag_must_val means that every tag is allowed, - * while a value of \c 0xFF for \p tag_must_mask means - * that \p tag_must_val is the only allowed tag. - * \param tag_may_mask A mask to be applied to the ASN.1 tags found within - * the SEQUENCE before comparing to \p tag_may_val. - * \param tag_may_val The desired value of each ASN.1 tag found in the - * SEQUENCE, after masking with \p tag_may_mask. - * Mismatching tags will be silently ignored. - * For example, a value of \c 0 for \p tag_may_mask and - * \p tag_may_val means that any tag will be considered, - * while a value of \c 0xFF for \p tag_may_mask means - * that all tags with value different from \p tag_may_val - * will be ignored. - * \param cb The callback to trigger for each component - * in the ASN.1 SEQUENCE that matches \p tag_may_val. - * The callback function is called with the following - * parameters: - * - \p ctx. - * - The tag of the current element. - * - A pointer to the start of the current element's - * content inside the input. - * - The length of the content of the current element. - * If the callback returns a non-zero value, - * the function stops immediately, - * forwarding the callback's return value. - * \param ctx The context to be passed to the callback \p cb. - * - * \return \c 0 if successful the entire ASN.1 SEQUENCE - * was traversed without parsing or callback errors. - * \return #MBEDTLS_ERR_ASN1_LENGTH_MISMATCH if the input - * contains extra data after a valid SEQUENCE - * of elements with an accepted tag. - * \return #MBEDTLS_ERR_ASN1_UNEXPECTED_TAG if the input starts - * with an ASN.1 SEQUENCE in which an element has a tag - * that is not accepted. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 SEQUENCE. - * \return A non-zero error code forwarded from the callback - * \p cb in case the latter returns a non-zero value. - */ -int mbedtls_asn1_traverse_sequence_of( - unsigned char **p, - const unsigned char *end, - unsigned char tag_must_mask, unsigned char tag_must_val, - unsigned char tag_may_mask, unsigned char tag_may_val, - int (*cb)(void *ctx, int tag, - unsigned char *start, size_t len), - void *ctx); - -#if defined(MBEDTLS_BIGNUM_C) -/** - * \brief Retrieve an integer ASN.1 tag and its value. - * Updates the pointer to immediately behind the full tag. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the ASN.1 element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param X On success, the parsed value. - * - * \return 0 if successful. - * \return An ASN.1 error code if the input does not start with - * a valid ASN.1 INTEGER. - * \return #MBEDTLS_ERR_ASN1_INVALID_LENGTH if the parsed value does - * not fit in an \c int. - * \return An MPI error code if the parsed value is too large. - */ -int mbedtls_asn1_get_mpi(unsigned char **p, - const unsigned char *end, - mbedtls_mpi *X); -#endif /* MBEDTLS_BIGNUM_C */ - -/** - * \brief Retrieve an AlgorithmIdentifier ASN.1 sequence. - * Updates the pointer to immediately behind the full - * AlgorithmIdentifier. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the AlgorithmIdentifier element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param alg The buffer to receive the OID. - * \param params The buffer to receive the parameters. - * This is zeroized if there are no parameters. - * - * \return 0 if successful or a specific ASN.1 or MPI error code. - */ -int mbedtls_asn1_get_alg(unsigned char **p, - const unsigned char *end, - mbedtls_asn1_buf *alg, mbedtls_asn1_buf *params); - -/** - * \brief Retrieve an AlgorithmIdentifier ASN.1 sequence with NULL or no - * params. - * Updates the pointer to immediately behind the full - * AlgorithmIdentifier. - * - * \param p On entry, \c *p points to the start of the ASN.1 element. - * On successful completion, \c *p points to the first byte - * beyond the AlgorithmIdentifier element. - * On error, the value of \c *p is undefined. - * \param end End of data. - * \param alg The buffer to receive the OID. - * - * \return 0 if successful or a specific ASN.1 or MPI error code. - */ -int mbedtls_asn1_get_alg_null(unsigned char **p, - const unsigned char *end, - mbedtls_asn1_buf *alg); - -/** - * \brief Find a specific named_data entry in a sequence or list based on - * the OID. - * - * \param list The list to seek through - * \param oid The OID to look for - * \param len Size of the OID - * - * \return NULL if not found, or a pointer to the existing entry. - */ -const mbedtls_asn1_named_data *mbedtls_asn1_find_named_data(const mbedtls_asn1_named_data *list, - const char *oid, size_t len); - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief Free a mbedtls_asn1_named_data entry - * - * \deprecated This function is deprecated and will be removed in a - * future version of the library. - * Please use mbedtls_asn1_free_named_data_list() - * or mbedtls_asn1_free_named_data_list_shallow(). - * - * \param entry The named data entry to free. - * This function calls mbedtls_free() on - * `entry->oid.p` and `entry->val.p`. - */ -void MBEDTLS_DEPRECATED mbedtls_asn1_free_named_data(mbedtls_asn1_named_data *entry); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/** - * \brief Free all entries in a mbedtls_asn1_named_data list. - * - * \param head Pointer to the head of the list of named data entries to free. - * This function calls mbedtls_free() on - * `entry->oid.p` and `entry->val.p` and then on `entry` - * for each list entry, and sets \c *head to \c NULL. - */ -void mbedtls_asn1_free_named_data_list(mbedtls_asn1_named_data **head); - -/** - * \brief Free all shallow entries in a mbedtls_asn1_named_data list, - * but do not free internal pointer targets. - * - * \param name Head of the list of named data entries to free. - * This function calls mbedtls_free() on each list element. - */ -void mbedtls_asn1_free_named_data_list_shallow(mbedtls_asn1_named_data *name); - -/** \} name Functions to parse ASN.1 data structures */ -/** \} addtogroup asn1_module */ - -#endif /* MBEDTLS_ASN1_PARSE_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* asn1.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/asn1write.h b/android/mbedtls/arm64-v8a/include/mbedtls/asn1write.h deleted file mode 100644 index 0c5a85ac2..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/asn1write.h +++ /dev/null @@ -1,390 +0,0 @@ -/** - * \file asn1write.h - * - * \brief ASN.1 buffer writing functionality - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_ASN1_WRITE_H -#define MBEDTLS_ASN1_WRITE_H - -#include "mbedtls/build_info.h" - -#include "mbedtls/asn1.h" - -#define MBEDTLS_ASN1_CHK_ADD(g, f) \ - do \ - { \ - if ((ret = (f)) < 0) \ - return ret; \ - else \ - (g) += ret; \ - } while (0) - -#define MBEDTLS_ASN1_CHK_CLEANUP_ADD(g, f) \ - do \ - { \ - if ((ret = (f)) < 0) \ - goto cleanup; \ - else \ - (g) += ret; \ - } while (0) - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(MBEDTLS_ASN1_WRITE_C) || defined(MBEDTLS_X509_USE_C) || \ - defined(MBEDTLS_PSA_UTIL_HAVE_ECDSA) -/** - * \brief Write a length field in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param len The length value to write. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_len(unsigned char **p, const unsigned char *start, - size_t len); -/** - * \brief Write an ASN.1 tag in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param tag The tag to write. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_tag(unsigned char **p, const unsigned char *start, - unsigned char tag); -#endif /* MBEDTLS_ASN1_WRITE_C || MBEDTLS_X509_USE_C || MBEDTLS_PSA_UTIL_HAVE_ECDSA*/ - -#if defined(MBEDTLS_ASN1_WRITE_C) -/** - * \brief Write raw buffer data. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param buf The data buffer to write. - * \param size The length of the data buffer. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_raw_buffer(unsigned char **p, const unsigned char *start, - const unsigned char *buf, size_t size); - -#if defined(MBEDTLS_BIGNUM_C) -/** - * \brief Write an arbitrary-precision number (#MBEDTLS_ASN1_INTEGER) - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param X The MPI to write. - * It must be non-negative. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_mpi(unsigned char **p, const unsigned char *start, - const mbedtls_mpi *X); -#endif /* MBEDTLS_BIGNUM_C */ - -/** - * \brief Write a NULL tag (#MBEDTLS_ASN1_NULL) with zero data - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_null(unsigned char **p, const unsigned char *start); - -/** - * \brief Write an OID tag (#MBEDTLS_ASN1_OID) and data - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param oid The OID to write. - * \param oid_len The length of the OID. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_oid(unsigned char **p, const unsigned char *start, - const char *oid, size_t oid_len); - -/** - * \brief Write an AlgorithmIdentifier sequence in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param oid The OID of the algorithm to write. - * \param oid_len The length of the algorithm's OID. - * \param par_len The length of the parameters, which must be already written. - * If 0, NULL parameters are added - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_algorithm_identifier(unsigned char **p, - const unsigned char *start, - const char *oid, size_t oid_len, - size_t par_len); - -/** - * \brief Write an AlgorithmIdentifier sequence in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param oid The OID of the algorithm to write. - * \param oid_len The length of the algorithm's OID. - * \param par_len The length of the parameters, which must be already written. - * \param has_par If there are any parameters. If 0, par_len must be 0. If 1 - * and \p par_len is 0, NULL parameters are added. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_algorithm_identifier_ext(unsigned char **p, - const unsigned char *start, - const char *oid, size_t oid_len, - size_t par_len, int has_par); - -/** - * \brief Write a boolean tag (#MBEDTLS_ASN1_BOOLEAN) and value - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param boolean The boolean value to write, either \c 0 or \c 1. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_bool(unsigned char **p, const unsigned char *start, - int boolean); - -/** - * \brief Write an int tag (#MBEDTLS_ASN1_INTEGER) and value - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param val The integer value to write. - * It must be non-negative. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_int(unsigned char **p, const unsigned char *start, int val); - -/** - * \brief Write an enum tag (#MBEDTLS_ASN1_ENUMERATED) and value - * in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param val The integer value to write. - * - * \return The number of bytes written to \p p on success. - * \return A negative \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_asn1_write_enum(unsigned char **p, const unsigned char *start, int val); - -/** - * \brief Write a string in ASN.1 format using a specific - * string encoding tag. - - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param tag The string encoding tag to write, e.g. - * #MBEDTLS_ASN1_UTF8_STRING. - * \param text The string to write. - * \param text_len The length of \p text in bytes (which might - * be strictly larger than the number of characters). - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_tagged_string(unsigned char **p, const unsigned char *start, - int tag, const char *text, - size_t text_len); - -/** - * \brief Write a string in ASN.1 format using the PrintableString - * string encoding tag (#MBEDTLS_ASN1_PRINTABLE_STRING). - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param text The string to write. - * \param text_len The length of \p text in bytes (which might - * be strictly larger than the number of characters). - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_printable_string(unsigned char **p, - const unsigned char *start, - const char *text, size_t text_len); - -/** - * \brief Write a UTF8 string in ASN.1 format using the UTF8String - * string encoding tag (#MBEDTLS_ASN1_UTF8_STRING). - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param text The string to write. - * \param text_len The length of \p text in bytes (which might - * be strictly larger than the number of characters). - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_utf8_string(unsigned char **p, const unsigned char *start, - const char *text, size_t text_len); - -/** - * \brief Write a string in ASN.1 format using the IA5String - * string encoding tag (#MBEDTLS_ASN1_IA5_STRING). - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param text The string to write. - * \param text_len The length of \p text in bytes (which might - * be strictly larger than the number of characters). - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_ia5_string(unsigned char **p, const unsigned char *start, - const char *text, size_t text_len); - -/** - * \brief Write a bitstring tag (#MBEDTLS_ASN1_BIT_STRING) and - * value in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param buf The bitstring to write. - * \param bits The total number of bits in the bitstring. - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_bitstring(unsigned char **p, const unsigned char *start, - const unsigned char *buf, size_t bits); - -/** - * \brief This function writes a named bitstring tag - * (#MBEDTLS_ASN1_BIT_STRING) and value in ASN.1 format. - * - * As stated in RFC 5280 Appendix B, trailing zeroes are - * omitted when encoding named bitstrings in DER. - * - * \note This function works backwards within the data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer which is used for bounds-checking. - * \param buf The bitstring to write. - * \param bits The total number of bits in the bitstring. - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_named_bitstring(unsigned char **p, - const unsigned char *start, - const unsigned char *buf, - size_t bits); - -/** - * \brief Write an octet string tag (#MBEDTLS_ASN1_OCTET_STRING) - * and value in ASN.1 format. - * - * \note This function works backwards in data buffer. - * - * \param p The reference to the current position pointer. - * \param start The start of the buffer, for bounds-checking. - * \param buf The buffer holding the data to write. - * \param size The length of the data buffer \p buf. - * - * \return The number of bytes written to \p p on success. - * \return A negative error code on failure. - */ -int mbedtls_asn1_write_octet_string(unsigned char **p, const unsigned char *start, - const unsigned char *buf, size_t size); - -/** - * \brief Create or find a specific named_data entry for writing in a - * sequence or list based on the OID. If not already in there, - * a new entry is added to the head of the list. - * Warning: Destructive behaviour for the val data! - * - * \param list The pointer to the location of the head of the list to seek - * through (will be updated in case of a new entry). - * \param oid The OID to look for. - * \param oid_len The size of the OID. - * \param val The associated data to store. If this is \c NULL, - * no data is copied to the new or existing buffer. - * \param val_len The minimum length of the data buffer needed. - * If this is 0, do not allocate a buffer for the associated - * data. - * If the OID was already present, enlarge, shrink or free - * the existing buffer to fit \p val_len. - * - * \return A pointer to the new / existing entry on success. - * \return \c NULL if there was a memory allocation error. - */ -mbedtls_asn1_named_data *mbedtls_asn1_store_named_data(mbedtls_asn1_named_data **list, - const char *oid, size_t oid_len, - const unsigned char *val, - size_t val_len); - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_ASN1_WRITE_C */ - -#endif /* MBEDTLS_ASN1_WRITE_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/base64.h b/android/mbedtls/arm64-v8a/include/mbedtls/base64.h deleted file mode 100644 index 8f459b74c..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/base64.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - * \file base64.h - * - * \brief RFC 1521 base64 encoding/decoding - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_BASE64_H -#define MBEDTLS_BASE64_H - -#include "mbedtls/build_info.h" - -#include - -/** Output buffer too small. */ -#define MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL -0x002A -/** Invalid character in input. */ -#define MBEDTLS_ERR_BASE64_INVALID_CHARACTER -0x002C - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Encode a buffer into base64 format - * - * \param dst destination buffer - * \param dlen size of the destination buffer - * \param olen number of bytes written - * \param src source buffer - * \param slen amount of data to be encoded - * - * \return 0 if successful, or MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL. - * *olen is always updated to reflect the amount - * of data that has (or would have) been written. - * If that length cannot be represented, then no data is - * written to the buffer and *olen is set to the maximum - * length representable as a size_t. - * - * \note Call this function with dlen = 0 to obtain the - * required buffer size in *olen - */ -int mbedtls_base64_encode(unsigned char *dst, size_t dlen, size_t *olen, - const unsigned char *src, size_t slen); - -/** - * \brief Decode a base64-formatted buffer - * - * \param dst destination buffer (can be NULL for checking size) - * \param dlen size of the destination buffer - * \param olen number of bytes written - * \param src source buffer - * \param slen amount of data to be decoded - * - * \return 0 if successful, MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL, or - * MBEDTLS_ERR_BASE64_INVALID_CHARACTER if the input data is - * not correct. *olen is always updated to reflect the amount - * of data that has (or would have) been written. - * - * \note Call this function with *dst = NULL or dlen = 0 to obtain - * the required buffer size in *olen - */ -int mbedtls_base64_decode(unsigned char *dst, size_t dlen, size_t *olen, - const unsigned char *src, size_t slen); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -int mbedtls_base64_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* base64.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/bignum.h b/android/mbedtls/arm64-v8a/include/mbedtls/bignum.h deleted file mode 100644 index 618785671..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/bignum.h +++ /dev/null @@ -1,1088 +0,0 @@ -/** - * \file bignum.h - * - * \brief Multi-precision integer library - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_BIGNUM_H -#define MBEDTLS_BIGNUM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include -#include - -#if defined(MBEDTLS_FS_IO) -#include -#endif - -/** An error occurred while reading from or writing to a file. */ -#define MBEDTLS_ERR_MPI_FILE_IO_ERROR -0x0002 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_MPI_BAD_INPUT_DATA -0x0004 -/** There is an invalid character in the digit string. */ -#define MBEDTLS_ERR_MPI_INVALID_CHARACTER -0x0006 -/** The buffer is too small to write to. */ -#define MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL -0x0008 -/** The input arguments are negative or result in illegal output. */ -#define MBEDTLS_ERR_MPI_NEGATIVE_VALUE -0x000A -/** The input argument for division is zero, which is not allowed. */ -#define MBEDTLS_ERR_MPI_DIVISION_BY_ZERO -0x000C -/** The input arguments are not acceptable. */ -#define MBEDTLS_ERR_MPI_NOT_ACCEPTABLE -0x000E -/** Memory allocation failed. */ -#define MBEDTLS_ERR_MPI_ALLOC_FAILED -0x0010 - -#define MBEDTLS_MPI_CHK(f) \ - do \ - { \ - if ((ret = (f)) != 0) \ - goto cleanup; \ - } while (0) - -/* - * Maximum size MPIs are allowed to grow to in number of limbs. - */ -#define MBEDTLS_MPI_MAX_LIMBS 10000 - -#if !defined(MBEDTLS_MPI_WINDOW_SIZE) -/* - * Maximum window size used for modular exponentiation. Default: 3 - * Minimum value: 1. Maximum value: 6. - * - * Result is an array of ( 2 ** MBEDTLS_MPI_WINDOW_SIZE ) MPIs used - * for the sliding window calculation. (So 8 by default) - * - * Reduction in size, reduces speed. - */ -#define MBEDTLS_MPI_WINDOW_SIZE 3 /**< Maximum window size used. */ -#endif /* !MBEDTLS_MPI_WINDOW_SIZE */ - -#if !defined(MBEDTLS_MPI_MAX_SIZE) -/* - * Maximum size of MPIs allowed in bits and bytes for user-MPIs. - * ( Default: 512 bytes => 4096 bits, Maximum tested: 2048 bytes => 16384 bits ) - * - * Note: Calculations can temporarily result in larger MPIs. So the number - * of limbs required (MBEDTLS_MPI_MAX_LIMBS) is higher. - */ -#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ -#endif /* !MBEDTLS_MPI_MAX_SIZE */ - -#define MBEDTLS_MPI_MAX_BITS (8 * MBEDTLS_MPI_MAX_SIZE) /**< Maximum number of bits for usable MPIs. */ - -/* - * When reading from files with mbedtls_mpi_read_file() and writing to files with - * mbedtls_mpi_write_file() the buffer should have space - * for a (short) label, the MPI (in the provided radix), the newline - * characters and the '\0'. - * - * By default we assume at least a 10 char label, a minimum radix of 10 - * (decimal) and a maximum of 4096 bit numbers (1234 decimal chars). - * Autosized at compile time for at least a 10 char label, a minimum radix - * of 10 (decimal) for a number of MBEDTLS_MPI_MAX_BITS size. - * - * This used to be statically sized to 1250 for a maximum of 4096 bit - * numbers (1234 decimal chars). - * - * Calculate using the formula: - * MBEDTLS_MPI_RW_BUFFER_SIZE = ceil(MBEDTLS_MPI_MAX_BITS / ln(10) * ln(2)) + - * LabelSize + 6 - */ -#define MBEDTLS_MPI_MAX_BITS_SCALE100 (100 * MBEDTLS_MPI_MAX_BITS) -#define MBEDTLS_LN_2_DIV_LN_10_SCALE100 332 -#define MBEDTLS_MPI_RW_BUFFER_SIZE (((MBEDTLS_MPI_MAX_BITS_SCALE100 + \ - MBEDTLS_LN_2_DIV_LN_10_SCALE100 - 1) / \ - MBEDTLS_LN_2_DIV_LN_10_SCALE100) + 10 + 6) - -/* - * Define the base integer type, architecture-wise. - * - * 32 or 64-bit integer types can be forced regardless of the underlying - * architecture by defining MBEDTLS_HAVE_INT32 or MBEDTLS_HAVE_INT64 - * respectively and undefining MBEDTLS_HAVE_ASM. - * - * Double-width integers (e.g. 128-bit in 64-bit architectures) can be - * disabled by defining MBEDTLS_NO_UDBL_DIVISION. - */ -#if !defined(MBEDTLS_HAVE_INT32) - #if defined(_MSC_VER) && defined(_M_AMD64) -/* Always choose 64-bit when using MSC */ - #if !defined(MBEDTLS_HAVE_INT64) - #define MBEDTLS_HAVE_INT64 - #endif /* !MBEDTLS_HAVE_INT64 */ -typedef int64_t mbedtls_mpi_sint; -typedef uint64_t mbedtls_mpi_uint; -#define MBEDTLS_MPI_UINT_MAX UINT64_MAX - #elif defined(__GNUC__) && ( \ - defined(__amd64__) || defined(__x86_64__) || \ - defined(__ppc64__) || defined(__powerpc64__) || \ - defined(__ia64__) || defined(__alpha__) || \ - (defined(__sparc__) && defined(__arch64__)) || \ - defined(__s390x__) || defined(__mips64) || \ - defined(__aarch64__)) - #if !defined(MBEDTLS_HAVE_INT64) - #define MBEDTLS_HAVE_INT64 - #endif /* MBEDTLS_HAVE_INT64 */ -typedef int64_t mbedtls_mpi_sint; -typedef uint64_t mbedtls_mpi_uint; -#define MBEDTLS_MPI_UINT_MAX UINT64_MAX - #if !defined(MBEDTLS_NO_UDBL_DIVISION) -/* mbedtls_t_udbl defined as 128-bit unsigned int */ -typedef unsigned int mbedtls_t_udbl __attribute__((mode(TI))); - #define MBEDTLS_HAVE_UDBL - #endif /* !MBEDTLS_NO_UDBL_DIVISION */ - #elif defined(__ARMCC_VERSION) && defined(__aarch64__) -/* - * __ARMCC_VERSION is defined for both armcc and armclang and - * __aarch64__ is only defined by armclang when compiling 64-bit code - */ - #if !defined(MBEDTLS_HAVE_INT64) - #define MBEDTLS_HAVE_INT64 - #endif /* !MBEDTLS_HAVE_INT64 */ -typedef int64_t mbedtls_mpi_sint; -typedef uint64_t mbedtls_mpi_uint; -#define MBEDTLS_MPI_UINT_MAX UINT64_MAX - #if !defined(MBEDTLS_NO_UDBL_DIVISION) -/* mbedtls_t_udbl defined as 128-bit unsigned int */ -typedef __uint128_t mbedtls_t_udbl; - #define MBEDTLS_HAVE_UDBL - #endif /* !MBEDTLS_NO_UDBL_DIVISION */ - #elif defined(MBEDTLS_HAVE_INT64) -/* Force 64-bit integers with unknown compiler */ -typedef int64_t mbedtls_mpi_sint; -typedef uint64_t mbedtls_mpi_uint; -#define MBEDTLS_MPI_UINT_MAX UINT64_MAX - #endif -#endif /* !MBEDTLS_HAVE_INT32 */ - -#if !defined(MBEDTLS_HAVE_INT64) -/* Default to 32-bit compilation */ - #if !defined(MBEDTLS_HAVE_INT32) - #define MBEDTLS_HAVE_INT32 - #endif /* !MBEDTLS_HAVE_INT32 */ -typedef int32_t mbedtls_mpi_sint; -typedef uint32_t mbedtls_mpi_uint; -#define MBEDTLS_MPI_UINT_MAX UINT32_MAX - #if !defined(MBEDTLS_NO_UDBL_DIVISION) -typedef uint64_t mbedtls_t_udbl; - #define MBEDTLS_HAVE_UDBL - #endif /* !MBEDTLS_NO_UDBL_DIVISION */ -#endif /* !MBEDTLS_HAVE_INT64 */ - -/* - * Sanity check that exactly one of MBEDTLS_HAVE_INT32 or MBEDTLS_HAVE_INT64 is defined, - * so that code elsewhere doesn't have to check. - */ -#if (!(defined(MBEDTLS_HAVE_INT32) || defined(MBEDTLS_HAVE_INT64))) || \ - (defined(MBEDTLS_HAVE_INT32) && defined(MBEDTLS_HAVE_INT64)) -#error "Only 32-bit or 64-bit limbs are supported in bignum" -#endif - -/** \typedef mbedtls_mpi_uint - * \brief The type of machine digits in a bignum, called _limbs_. - * - * This is always an unsigned integer type with no padding bits. The size - * is platform-dependent. - */ - -/** \typedef mbedtls_mpi_sint - * \brief The signed type corresponding to #mbedtls_mpi_uint. - * - * This is always an signed integer type with no padding bits. The size - * is platform-dependent. - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief MPI structure - */ -typedef struct mbedtls_mpi { - /** Pointer to limbs. - * - * This may be \c NULL if \c n is 0. - */ - mbedtls_mpi_uint *MBEDTLS_PRIVATE(p); - - /** Sign: -1 if the mpi is negative, 1 otherwise. - * - * The number 0 must be represented with `s = +1`. Although many library - * functions treat all-limbs-zero as equivalent to a valid representation - * of 0 regardless of the sign bit, there are exceptions, so bignum - * functions and external callers must always set \c s to +1 for the - * number zero. - * - * Note that this implies that calloc() or `... = {0}` does not create - * a valid MPI representation. You must call mbedtls_mpi_init(). - */ - signed short MBEDTLS_PRIVATE(s); - - /** Total number of limbs in \c p. */ - unsigned short MBEDTLS_PRIVATE(n); - /* Make sure that MBEDTLS_MPI_MAX_LIMBS fits in n. - * Use the same limit value on all platforms so that we don't have to - * think about different behavior on the rare platforms where - * unsigned short can store values larger than the minimum required by - * the C language, which is 65535. - */ -#if MBEDTLS_MPI_MAX_LIMBS > 65535 -#error "MBEDTLS_MPI_MAX_LIMBS > 65535 is not supported" -#endif -} -mbedtls_mpi; - -/** - * \brief Initialize an MPI context. - * - * This makes the MPI ready to be set or freed, - * but does not define a value for the MPI. - * - * \param X The MPI context to initialize. This must not be \c NULL. - */ -void mbedtls_mpi_init(mbedtls_mpi *X); - -/** - * \brief This function frees the components of an MPI context. - * - * \param X The MPI context to be cleared. This may be \c NULL, - * in which case this function is a no-op. If it is - * not \c NULL, it must point to an initialized MPI. - */ -void mbedtls_mpi_free(mbedtls_mpi *X); - -/** - * \brief Enlarge an MPI to the specified number of limbs. - * - * \note This function does nothing if the MPI is - * already large enough. - * - * \param X The MPI to grow. It must be initialized. - * \param nblimbs The target number of limbs. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_grow(mbedtls_mpi *X, size_t nblimbs); - -/** - * \brief This function resizes an MPI downwards, keeping at least the - * specified number of limbs. - * - * If \c X is smaller than \c nblimbs, it is resized up - * instead. - * - * \param X The MPI to shrink. This must point to an initialized MPI. - * \param nblimbs The minimum number of limbs to keep. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed - * (this can only happen when resizing up). - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_shrink(mbedtls_mpi *X, size_t nblimbs); - -/** - * \brief Make a copy of an MPI. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param Y The source MPI. This must point to an initialized MPI. - * - * \note The limb-buffer in the destination MPI is enlarged - * if necessary to hold the value in the source MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_copy(mbedtls_mpi *X, const mbedtls_mpi *Y); - -/** - * \brief Swap the contents of two MPIs. - * - * \param X The first MPI. It must be initialized. - * \param Y The second MPI. It must be initialized. - */ -void mbedtls_mpi_swap(mbedtls_mpi *X, mbedtls_mpi *Y); - -/** - * \brief Perform a safe conditional copy of MPI which doesn't - * reveal whether the condition was true or not. - * - * \param X The MPI to conditionally assign to. This must point - * to an initialized MPI. - * \param Y The MPI to be assigned from. This must point to an - * initialized MPI. - * \param assign The condition deciding whether to perform the - * assignment or not. Must be either 0 or 1: - * * \c 1: Perform the assignment `X = Y`. - * * \c 0: Keep the original value of \p X. - * - * \note This function is equivalent to - * `if( assign ) mbedtls_mpi_copy( X, Y );` - * except that it avoids leaking any information about whether - * the assignment was done or not (the above code may leak - * information through branch prediction and/or memory access - * patterns analysis). - * - * \warning If \p assign is neither 0 nor 1, the result of this function - * is indeterminate, and the resulting value in \p X might be - * neither its original value nor the value in \p Y. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_safe_cond_assign(mbedtls_mpi *X, const mbedtls_mpi *Y, unsigned char assign); - -/** - * \brief Perform a safe conditional swap which doesn't - * reveal whether the condition was true or not. - * - * \param X The first MPI. This must be initialized. - * \param Y The second MPI. This must be initialized. - * \param swap The condition deciding whether to perform - * the swap or not. Must be either 0 or 1: - * * \c 1: Swap the values of \p X and \p Y. - * * \c 0: Keep the original values of \p X and \p Y. - * - * \note This function is equivalent to - * if( swap ) mbedtls_mpi_swap( X, Y ); - * except that it avoids leaking any information about whether - * the swap was done or not (the above code may leak - * information through branch prediction and/or memory access - * patterns analysis). - * - * \warning If \p swap is neither 0 nor 1, the result of this function - * is indeterminate, and both \p X and \p Y might end up with - * values different to either of the original ones. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - * - */ -int mbedtls_mpi_safe_cond_swap(mbedtls_mpi *X, mbedtls_mpi *Y, unsigned char swap); - -/** - * \brief Store integer value in MPI. - * - * \param X The MPI to set. This must be initialized. - * \param z The value to use. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_lset(mbedtls_mpi *X, mbedtls_mpi_sint z); - -/** - * \brief Get a specific bit from an MPI. - * - * \param X The MPI to query. This must be initialized. - * \param pos Zero-based index of the bit to query. - * - * \return \c 0 or \c 1 on success, depending on whether bit \c pos - * of \c X is unset or set. - * \return A negative error code on failure. - */ -int mbedtls_mpi_get_bit(const mbedtls_mpi *X, size_t pos); - -/** - * \brief Modify a specific bit in an MPI. - * - * \note This function will grow the target MPI if necessary to set a - * bit to \c 1 in a not yet existing limb. It will not grow if - * the bit should be set to \c 0. - * - * \param X The MPI to modify. This must be initialized. - * \param pos Zero-based index of the bit to modify. - * \param val The desired value of bit \c pos: \c 0 or \c 1. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_set_bit(mbedtls_mpi *X, size_t pos, unsigned char val); - -/** - * \brief Return the number of bits of value \c 0 before the - * least significant bit of value \c 1. - * - * \note This is the same as the zero-based index of - * the least significant bit of value \c 1. - * - * \param X The MPI to query. - * - * \return The number of bits of value \c 0 before the least significant - * bit of value \c 1 in \p X. - */ -size_t mbedtls_mpi_lsb(const mbedtls_mpi *X); - -/** - * \brief Return the number of bits up to and including the most - * significant bit of value \c 1. - * - * * \note This is same as the one-based index of the most - * significant bit of value \c 1. - * - * \param X The MPI to query. This must point to an initialized MPI. - * - * \return The number of bits up to and including the most - * significant bit of value \c 1. - */ -size_t mbedtls_mpi_bitlen(const mbedtls_mpi *X); - -/** - * \brief Return the total size of an MPI value in bytes. - * - * \param X The MPI to use. This must point to an initialized MPI. - * - * \note The value returned by this function may be less than - * the number of bytes used to store \p X internally. - * This happens if and only if there are trailing bytes - * of value zero. - * - * \return The least number of bytes capable of storing - * the absolute value of \p X. - */ -size_t mbedtls_mpi_size(const mbedtls_mpi *X); - -/** - * \brief Import an MPI from an ASCII string. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param radix The numeric base of the input string. - * \param s Null-terminated string buffer. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_mpi_read_string(mbedtls_mpi *X, int radix, const char *s); - -/** - * \brief Export an MPI to an ASCII string. - * - * \param X The source MPI. This must point to an initialized MPI. - * \param radix The numeric base of the output string. - * \param buf The buffer to write the string to. This must be writable - * buffer of length \p buflen Bytes. - * \param buflen The available size in Bytes of \p buf. - * \param olen The address at which to store the length of the string - * written, including the final \c NULL byte. This must - * not be \c NULL. - * - * \note You can call this function with `buflen == 0` to obtain the - * minimum required buffer size in `*olen`. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL if the target buffer \p buf - * is too small to hold the value of \p X in the desired base. - * In this case, `*olen` is nonetheless updated to contain the - * size of \p buf required for a successful call. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_write_string(const mbedtls_mpi *X, int radix, - char *buf, size_t buflen, size_t *olen); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief Read an MPI from a line in an opened file. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param radix The numeric base of the string representation used - * in the source line. - * \param fin The input file handle to use. This must not be \c NULL. - * - * \note On success, this function advances the file stream - * to the end of the current line or to EOF. - * - * The function returns \c 0 on an empty line. - * - * Leading whitespaces are ignored, as is a - * '0x' prefix for radix \c 16. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL if the file read buffer - * is too small. - * \return Another negative error code on failure. - */ -int mbedtls_mpi_read_file(mbedtls_mpi *X, int radix, FILE *fin); - -/** - * \brief Export an MPI into an opened file. - * - * \param p A string prefix to emit prior to the MPI data. - * For example, this might be a label, or "0x" when - * printing in base \c 16. This may be \c NULL if no prefix - * is needed. - * \param X The source MPI. This must point to an initialized MPI. - * \param radix The numeric base to be used in the emitted string. - * \param fout The output file handle. This may be \c NULL, in which case - * the output is written to \c stdout. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_mpi_write_file(const char *p, const mbedtls_mpi *X, - int radix, FILE *fout); -#endif /* MBEDTLS_FS_IO */ - -/** - * \brief Import an MPI from unsigned big endian binary data. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param buf The input buffer. This must be a readable buffer of length - * \p buflen Bytes. - * \param buflen The length of the input buffer \p buf in Bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_read_binary(mbedtls_mpi *X, const unsigned char *buf, - size_t buflen); - -/** - * \brief Import X from unsigned binary data, little endian - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param buf The input buffer. This must be a readable buffer of length - * \p buflen Bytes. - * \param buflen The length of the input buffer \p buf in Bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_read_binary_le(mbedtls_mpi *X, - const unsigned char *buf, size_t buflen); - -/** - * \brief Export X into unsigned binary data, big endian. - * Always fills the whole buffer, which will start with zeros - * if the number is smaller. - * - * \param X The source MPI. This must point to an initialized MPI. - * \param buf The output buffer. This must be a writable buffer of length - * \p buflen Bytes. - * \param buflen The size of the output buffer \p buf in Bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL if \p buf isn't - * large enough to hold the value of \p X. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_write_binary(const mbedtls_mpi *X, unsigned char *buf, - size_t buflen); - -/** - * \brief Export X into unsigned binary data, little endian. - * Always fills the whole buffer, which will end with zeros - * if the number is smaller. - * - * \param X The source MPI. This must point to an initialized MPI. - * \param buf The output buffer. This must be a writable buffer of length - * \p buflen Bytes. - * \param buflen The size of the output buffer \p buf in Bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL if \p buf isn't - * large enough to hold the value of \p X. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_write_binary_le(const mbedtls_mpi *X, - unsigned char *buf, size_t buflen); - -/** - * \brief Perform a left-shift on an MPI: X <<= count - * - * \param X The MPI to shift. This must point to an initialized MPI. - * The MPI pointed by \p X may be resized to fit - * the resulting number. - * \param count The number of bits to shift by. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_shift_l(mbedtls_mpi *X, size_t count); - -/** - * \brief Perform a right-shift on an MPI: X >>= count - * - * \param X The MPI to shift. This must point to an initialized MPI. - * \param count The number of bits to shift by. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_shift_r(mbedtls_mpi *X, size_t count); - -/** - * \brief Compare the absolute values of two MPIs. - * - * \param X The left-hand MPI. This must point to an initialized MPI. - * \param Y The right-hand MPI. This must point to an initialized MPI. - * - * \return \c 1 if `|X|` is greater than `|Y|`. - * \return \c -1 if `|X|` is lesser than `|Y|`. - * \return \c 0 if `|X|` is equal to `|Y|`. - */ -int mbedtls_mpi_cmp_abs(const mbedtls_mpi *X, const mbedtls_mpi *Y); - -/** - * \brief Compare two MPIs. - * - * \param X The left-hand MPI. This must point to an initialized MPI. - * \param Y The right-hand MPI. This must point to an initialized MPI. - * - * \return \c 1 if \p X is greater than \p Y. - * \return \c -1 if \p X is lesser than \p Y. - * \return \c 0 if \p X is equal to \p Y. - */ -int mbedtls_mpi_cmp_mpi(const mbedtls_mpi *X, const mbedtls_mpi *Y); - -/** - * \brief Check if an MPI is less than the other in constant time. - * - * \param X The left-hand MPI. This must point to an initialized MPI - * with the same allocated length as Y. - * \param Y The right-hand MPI. This must point to an initialized MPI - * with the same allocated length as X. - * \param ret The result of the comparison: - * \c 1 if \p X is less than \p Y. - * \c 0 if \p X is greater than or equal to \p Y. - * - * \return 0 on success. - * \return MBEDTLS_ERR_MPI_BAD_INPUT_DATA if the allocated length of - * the two input MPIs is not the same. - */ -int mbedtls_mpi_lt_mpi_ct(const mbedtls_mpi *X, const mbedtls_mpi *Y, - unsigned *ret); - -/** - * \brief Compare an MPI with an integer. - * - * \param X The left-hand MPI. This must point to an initialized MPI. - * \param z The integer value to compare \p X to. - * - * \return \c 1 if \p X is greater than \p z. - * \return \c -1 if \p X is lesser than \p z. - * \return \c 0 if \p X is equal to \p z. - */ -int mbedtls_mpi_cmp_int(const mbedtls_mpi *X, mbedtls_mpi_sint z); - -/** - * \brief Perform an unsigned addition of MPIs: X = |A| + |B| - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The first summand. This must point to an initialized MPI. - * \param B The second summand. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_add_abs(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform an unsigned subtraction of MPIs: X = |A| - |B| - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The minuend. This must point to an initialized MPI. - * \param B The subtrahend. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_NEGATIVE_VALUE if \p B is greater than \p A. - * \return Another negative error code on different kinds of failure. - * - */ -int mbedtls_mpi_sub_abs(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a signed addition of MPIs: X = A + B - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The first summand. This must point to an initialized MPI. - * \param B The second summand. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_add_mpi(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a signed subtraction of MPIs: X = A - B - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The minuend. This must point to an initialized MPI. - * \param B The subtrahend. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_sub_mpi(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a signed addition of an MPI and an integer: X = A + b - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The first summand. This must point to an initialized MPI. - * \param b The second summand. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_add_int(mbedtls_mpi *X, const mbedtls_mpi *A, - mbedtls_mpi_sint b); - -/** - * \brief Perform a signed subtraction of an MPI and an integer: - * X = A - b - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The minuend. This must point to an initialized MPI. - * \param b The subtrahend. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_sub_int(mbedtls_mpi *X, const mbedtls_mpi *A, - mbedtls_mpi_sint b); - -/** - * \brief Perform a multiplication of two MPIs: X = A * B - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The first factor. This must point to an initialized MPI. - * \param B The second factor. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - * - */ -int mbedtls_mpi_mul_mpi(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a multiplication of an MPI with an unsigned integer: - * X = A * b - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param A The first factor. This must point to an initialized MPI. - * \param b The second factor. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - * - */ -int mbedtls_mpi_mul_int(mbedtls_mpi *X, const mbedtls_mpi *A, - mbedtls_mpi_uint b); - -/** - * \brief Perform a division with remainder of two MPIs: - * A = Q * B + R - * - * \param Q The destination MPI for the quotient. - * This may be \c NULL if the value of the - * quotient is not needed. This must not alias A or B. - * \param R The destination MPI for the remainder value. - * This may be \c NULL if the value of the - * remainder is not needed. This must not alias A or B. - * \param A The dividend. This must point to an initialized MPI. - * \param B The divisor. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return #MBEDTLS_ERR_MPI_DIVISION_BY_ZERO if \p B equals zero. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_div_mpi(mbedtls_mpi *Q, mbedtls_mpi *R, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a division with remainder of an MPI by an integer: - * A = Q * b + R - * - * \param Q The destination MPI for the quotient. - * This may be \c NULL if the value of the - * quotient is not needed. This must not alias A. - * \param R The destination MPI for the remainder value. - * This may be \c NULL if the value of the - * remainder is not needed. This must not alias A. - * \param A The dividend. This must point to an initialized MPi. - * \param b The divisor. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return #MBEDTLS_ERR_MPI_DIVISION_BY_ZERO if \p b equals zero. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_div_int(mbedtls_mpi *Q, mbedtls_mpi *R, const mbedtls_mpi *A, - mbedtls_mpi_sint b); - -/** - * \brief Perform a modular reduction. R = A mod B - * - * \param R The destination MPI for the residue value. - * This must point to an initialized MPI. - * \param A The MPI to compute the residue of. - * This must point to an initialized MPI. - * \param B The base of the modular reduction. - * This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_DIVISION_BY_ZERO if \p B equals zero. - * \return #MBEDTLS_ERR_MPI_NEGATIVE_VALUE if \p B is negative. - * \return Another negative error code on different kinds of failure. - * - */ -int mbedtls_mpi_mod_mpi(mbedtls_mpi *R, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Perform a modular reduction with respect to an integer. - * r = A mod b - * - * \param r The address at which to store the residue. - * This must not be \c NULL. - * \param A The MPI to compute the residue of. - * This must point to an initialized MPi. - * \param b The integer base of the modular reduction. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_DIVISION_BY_ZERO if \p b equals zero. - * \return #MBEDTLS_ERR_MPI_NEGATIVE_VALUE if \p b is negative. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_mod_int(mbedtls_mpi_uint *r, const mbedtls_mpi *A, - mbedtls_mpi_sint b); - -/** - * \brief Perform a modular exponentiation: X = A^E mod N - * - * \param X The destination MPI. This must point to an initialized MPI. - * This must not alias E or N. - * \param A The base of the exponentiation. - * This must point to an initialized MPI. - * \param E The exponent MPI. This must point to an initialized MPI. - * \param N The base for the modular reduction. This must point to an - * initialized MPI. - * \param prec_RR A helper MPI depending solely on \p N which can be used to - * speed-up multiple modular exponentiations for the same value - * of \p N. This may be \c NULL. If it is not \c NULL, it must - * point to an initialized MPI. If it hasn't been used after - * the call to mbedtls_mpi_init(), this function will compute - * the helper value and store it in \p prec_RR for reuse on - * subsequent calls to this function. Otherwise, the function - * will assume that \p prec_RR holds the helper value set by a - * previous call to mbedtls_mpi_exp_mod(), and reuse it. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_BAD_INPUT_DATA if \c N is negative or - * even, or if \c E is negative. - * \return Another negative error code on different kinds of failures. - * - */ -int mbedtls_mpi_exp_mod(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *E, const mbedtls_mpi *N, - mbedtls_mpi *prec_RR); - -/** - * \brief Fill an MPI with a number of random bytes. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param size The number of random bytes to generate. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on failure. - * - * \note The bytes obtained from the RNG are interpreted - * as a big-endian representation of an MPI; this can - * be relevant in applications like deterministic ECDSA. - */ -int mbedtls_mpi_fill_random(mbedtls_mpi *X, size_t size, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** Generate a random number uniformly in a range. - * - * This function generates a random number between \p min inclusive and - * \p N exclusive. - * - * The procedure complies with RFC 6979 §3.3 (deterministic ECDSA) - * when the RNG is a suitably parametrized instance of HMAC_DRBG - * and \p min is \c 1. - * - * \note There are `N - min` possible outputs. The lower bound - * \p min can be reached, but the upper bound \p N cannot. - * - * \param X The destination MPI. This must point to an initialized MPI. - * \param min The minimum value to return. - * It must be nonnegative. - * \param N The upper bound of the range, exclusive. - * In other words, this is one plus the maximum value to return. - * \p N must be strictly larger than \p min. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_BAD_INPUT_DATA if \p min or \p N is invalid - * or if they are incompatible. - * \return #MBEDTLS_ERR_MPI_NOT_ACCEPTABLE if the implementation was - * unable to find a suitable value within a limited number - * of attempts. This has a negligible probability if \p N - * is significantly larger than \p min, which is the case - * for all usual cryptographic applications. - * \return Another negative error code on failure. - */ -int mbedtls_mpi_random(mbedtls_mpi *X, - mbedtls_mpi_sint min, - const mbedtls_mpi *N, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Compute the greatest common divisor: G = gcd(A, B) - * - * \param G The destination MPI. This must point to an initialized MPI. - * This will always be positive or 0. - * \param A The first operand. This must point to an initialized MPI. - * \param B The second operand. This must point to an initialized MPI. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_mpi_gcd(mbedtls_mpi *G, const mbedtls_mpi *A, - const mbedtls_mpi *B); - -/** - * \brief Compute the modular inverse: X = A^-1 mod N - * - * \param X The destination MPI. This must point to an initialized MPI. - * The value returned on success will be between [1, N-1]. - * \param A The MPI to calculate the modular inverse of. This must point - * to an initialized MPI. This value can be negative, in which - * case a positive answer will still be returned in \p X. - * \param N The base of the modular inversion. This must point to an - * initialized MPI and be greater than one. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_BAD_INPUT_DATA if \p N is less than - * or equal to one. - * \return #MBEDTLS_ERR_MPI_NOT_ACCEPTABLE if \p A has no modular - * inverse with respect to \p N. - */ -int mbedtls_mpi_inv_mod(mbedtls_mpi *X, const mbedtls_mpi *A, - const mbedtls_mpi *N); - -/** - * \brief Miller-Rabin primality test. - * - * \warning If \p X is potentially generated by an adversary, for example - * when validating cryptographic parameters that you didn't - * generate yourself and that are supposed to be prime, then - * \p rounds should be at least the half of the security - * strength of the cryptographic algorithm. On the other hand, - * if \p X is chosen uniformly or non-adversarially (as is the - * case when mbedtls_mpi_gen_prime calls this function), then - * \p rounds can be much lower. - * - * \param X The MPI to check for primality. - * This must point to an initialized MPI. - * \param rounds The number of bases to perform the Miller-Rabin primality - * test for. The probability of returning 0 on a composite is - * at most 2-2*\p rounds . - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. - * This may be \c NULL if \p f_rng doesn't use - * a context parameter. - * - * \return \c 0 if successful, i.e. \p X is probably prime. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_NOT_ACCEPTABLE if \p X is not prime. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_mpi_is_prime_ext(const mbedtls_mpi *X, int rounds, - mbedtls_f_rng_t *f_rng, - void *p_rng); -/** - * \brief Flags for mbedtls_mpi_gen_prime() - * - * Each of these flags is a constraint on the result X returned by - * mbedtls_mpi_gen_prime(). - */ -typedef enum { - MBEDTLS_MPI_GEN_PRIME_FLAG_DH = 0x0001, /**< (X-1)/2 is prime too */ - MBEDTLS_MPI_GEN_PRIME_FLAG_LOW_ERR = 0x0002, /**< lower error rate from 2-80 to 2-128 */ -} mbedtls_mpi_gen_prime_flag_t; - -/** - * \brief Generate a prime number. - * - * \param X The destination MPI to store the generated prime in. - * This must point to an initialized MPi. - * \param nbits The required size of the destination MPI in bits. - * This must be between \c 3 and #MBEDTLS_MPI_MAX_BITS. - * \param flags A mask of flags of type #mbedtls_mpi_gen_prime_flag_t. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. - * This may be \c NULL if \p f_rng doesn't use - * a context parameter. - * - * \return \c 0 if successful, in which case \p X holds a - * probably prime number. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if a memory allocation failed. - * \return #MBEDTLS_ERR_MPI_BAD_INPUT_DATA if `nbits` is not between - * \c 3 and #MBEDTLS_MPI_MAX_BITS. - */ -int mbedtls_mpi_gen_prime(mbedtls_mpi *X, size_t nbits, int flags, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -int mbedtls_mpi_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* bignum.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/block_cipher.h b/android/mbedtls/arm64-v8a/include/mbedtls/block_cipher.h deleted file mode 100644 index 3f60f6f7d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/block_cipher.h +++ /dev/null @@ -1,76 +0,0 @@ -/** - * \file block_cipher.h - * - * \brief Internal abstraction layer. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_BLOCK_CIPHER_H -#define MBEDTLS_BLOCK_CIPHER_H - -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#if defined(MBEDTLS_AES_C) -#include "mbedtls/aes.h" -#endif -#if defined(MBEDTLS_ARIA_C) -#include "mbedtls/aria.h" -#endif -#if defined(MBEDTLS_CAMELLIA_C) -#include "mbedtls/camellia.h" -#endif - -#if defined(MBEDTLS_BLOCK_CIPHER_SOME_PSA) -#include "psa/crypto_types.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - MBEDTLS_BLOCK_CIPHER_ID_NONE = 0, /**< Unset. */ - MBEDTLS_BLOCK_CIPHER_ID_AES, /**< The AES cipher. */ - MBEDTLS_BLOCK_CIPHER_ID_CAMELLIA, /**< The Camellia cipher. */ - MBEDTLS_BLOCK_CIPHER_ID_ARIA, /**< The Aria cipher. */ -} mbedtls_block_cipher_id_t; - -/** - * Used internally to indicate whether a context uses legacy or PSA. - * - * Internal use only. - */ -typedef enum { - MBEDTLS_BLOCK_CIPHER_ENGINE_LEGACY = 0, - MBEDTLS_BLOCK_CIPHER_ENGINE_PSA, -} mbedtls_block_cipher_engine_t; - -typedef struct { - mbedtls_block_cipher_id_t MBEDTLS_PRIVATE(id); -#if defined(MBEDTLS_BLOCK_CIPHER_SOME_PSA) - mbedtls_block_cipher_engine_t MBEDTLS_PRIVATE(engine); - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(psa_key_id); -#endif - union { - unsigned dummy; /* Make the union non-empty even with no supported algorithms. */ -#if defined(MBEDTLS_AES_C) - mbedtls_aes_context MBEDTLS_PRIVATE(aes); -#endif -#if defined(MBEDTLS_ARIA_C) - mbedtls_aria_context MBEDTLS_PRIVATE(aria); -#endif -#if defined(MBEDTLS_CAMELLIA_C) - mbedtls_camellia_context MBEDTLS_PRIVATE(camellia); -#endif - } MBEDTLS_PRIVATE(ctx); -} mbedtls_block_cipher_context_t; - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_BLOCK_CIPHER_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/build_info.h b/android/mbedtls/arm64-v8a/include/mbedtls/build_info.h deleted file mode 100644 index 3d867aa1b..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/build_info.h +++ /dev/null @@ -1,194 +0,0 @@ -/** - * \file mbedtls/build_info.h - * - * \brief Build-time configuration info - * - * Include this file if you need to depend on the - * configuration options defined in mbedtls_config.h or MBEDTLS_CONFIG_FILE - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_BUILD_INFO_H -#define MBEDTLS_BUILD_INFO_H - -/* - * This set of compile-time defines can be used to determine the version number - * of the Mbed TLS library used. Run-time variables for the same can be found in - * version.h - */ - -/** - * The version number x.y.z is split into three parts. - * Major, Minor, Patchlevel - */ -#define MBEDTLS_VERSION_MAJOR 3 -#define MBEDTLS_VERSION_MINOR 6 -#define MBEDTLS_VERSION_PATCH 5 - -/** - * The single version number has the following structure: - * MMNNPP00 - * Major version | Minor version | Patch version - */ -#define MBEDTLS_VERSION_NUMBER 0x03060500 -#define MBEDTLS_VERSION_STRING "3.6.5" -#define MBEDTLS_VERSION_STRING_FULL "Mbed TLS 3.6.5" - -/* Macros for build-time platform detection */ - -#if !defined(MBEDTLS_ARCH_IS_ARM64) && \ - (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) -#define MBEDTLS_ARCH_IS_ARM64 -#endif - -#if !defined(MBEDTLS_ARCH_IS_ARM32) && \ - (defined(__arm__) || defined(_M_ARM) || \ - defined(_M_ARMT) || defined(__thumb__) || defined(__thumb2__)) -#define MBEDTLS_ARCH_IS_ARM32 -#endif - -#if !defined(MBEDTLS_ARCH_IS_X64) && \ - (defined(__amd64__) || defined(__x86_64__) || \ - ((defined(_M_X64) || defined(_M_AMD64)) && !defined(_M_ARM64EC))) -#define MBEDTLS_ARCH_IS_X64 -#endif - -#if !defined(MBEDTLS_ARCH_IS_X86) && \ - (defined(__i386__) || defined(_X86_) || \ - (defined(_M_IX86) && !defined(_M_I86))) -#define MBEDTLS_ARCH_IS_X86 -#endif - -#if !defined(MBEDTLS_PLATFORM_IS_WINDOWS_ON_ARM64) && \ - (defined(_M_ARM64) || defined(_M_ARM64EC)) -#define MBEDTLS_PLATFORM_IS_WINDOWS_ON_ARM64 -#endif - -/* This is defined if the architecture is Armv8-A, or higher */ -#if !defined(MBEDTLS_ARCH_IS_ARMV8_A) -#if defined(__ARM_ARCH) && defined(__ARM_ARCH_PROFILE) -#if (__ARM_ARCH >= 8) && (__ARM_ARCH_PROFILE == 'A') -/* GCC, clang, armclang and IAR */ -#define MBEDTLS_ARCH_IS_ARMV8_A -#endif -#elif defined(__ARM_ARCH_8A) -/* Alternative defined by clang */ -#define MBEDTLS_ARCH_IS_ARMV8_A -#elif defined(_M_ARM64) || defined(_M_ARM64EC) -/* MSVC ARM64 is at least Armv8.0-A */ -#define MBEDTLS_ARCH_IS_ARMV8_A -#endif -#endif - -#if defined(__GNUC__) && !defined(__ARMCC_VERSION) && !defined(__clang__) \ - && !defined(__llvm__) && !defined(__INTEL_COMPILER) -/* Defined if the compiler really is gcc and not clang, etc */ -#define MBEDTLS_COMPILER_IS_GCC -#define MBEDTLS_GCC_VERSION \ - (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -#endif - -#if defined(_MSC_VER) && !defined(_CRT_SECURE_NO_DEPRECATE) -#define _CRT_SECURE_NO_DEPRECATE 1 -#endif - -/* Define `inline` on some non-C99-compliant compilers. */ -#if (defined(__ARMCC_VERSION) || defined(_MSC_VER)) && \ - !defined(inline) && !defined(__cplusplus) -#define inline __inline -#endif - -#if defined(MBEDTLS_CONFIG_FILES_READ) -#error "Something went wrong: MBEDTLS_CONFIG_FILES_READ defined before reading the config files!" -#endif -#if defined(MBEDTLS_CONFIG_IS_FINALIZED) -#error "Something went wrong: MBEDTLS_CONFIG_IS_FINALIZED defined before reading the config files!" -#endif - -/* X.509, TLS and non-PSA crypto configuration */ -#if !defined(MBEDTLS_CONFIG_FILE) -#include "mbedtls/mbedtls_config.h" -#else -#include MBEDTLS_CONFIG_FILE -#endif - -#if defined(MBEDTLS_CONFIG_VERSION) && ( \ - MBEDTLS_CONFIG_VERSION < 0x03000000 || \ - MBEDTLS_CONFIG_VERSION > MBEDTLS_VERSION_NUMBER) -#error "Invalid config version, defined value of MBEDTLS_CONFIG_VERSION is unsupported" -#endif - -/* Target and application specific configurations - * - * Allow user to override any previous default. - * - */ -#if defined(MBEDTLS_USER_CONFIG_FILE) -#include MBEDTLS_USER_CONFIG_FILE -#endif - -/* PSA crypto configuration */ -#if defined(MBEDTLS_PSA_CRYPTO_CONFIG) -#if defined(MBEDTLS_PSA_CRYPTO_CONFIG_FILE) -#include MBEDTLS_PSA_CRYPTO_CONFIG_FILE -#else -#include "psa/crypto_config.h" -#endif -#if defined(MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE) -#include MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE -#endif -#endif /* defined(MBEDTLS_PSA_CRYPTO_CONFIG) */ - -/* Indicate that all configuration files have been read. - * It is now time to adjust the configuration (follow through on dependencies, - * make PSA and legacy crypto consistent, etc.). - */ -#define MBEDTLS_CONFIG_FILES_READ - -/* Auto-enable MBEDTLS_CTR_DRBG_USE_128_BIT_KEY if - * MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH and MBEDTLS_CTR_DRBG_C defined - * to ensure a 128-bit key size in CTR_DRBG. - */ -#if defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH) && defined(MBEDTLS_CTR_DRBG_C) -#define MBEDTLS_CTR_DRBG_USE_128_BIT_KEY -#endif - -/* Auto-enable MBEDTLS_MD_C if needed by a module that didn't require it - * in a previous release, to ensure backwards compatibility. - */ -#if defined(MBEDTLS_PKCS5_C) -#define MBEDTLS_MD_C -#endif - -/* PSA crypto specific configuration options - * - If config_psa.h reads a configuration option in preprocessor directive, - * this symbol should be set before its inclusion. (e.g. MBEDTLS_MD_C) - * - If config_psa.h writes a configuration option in conditional directive, - * this symbol should be consulted after its inclusion. - * (e.g. MBEDTLS_MD_LIGHT) - */ -#if defined(MBEDTLS_PSA_CRYPTO_CONFIG) /* PSA_WANT_xxx influences MBEDTLS_xxx */ || \ - defined(MBEDTLS_PSA_CRYPTO_C) /* MBEDTLS_xxx influences PSA_WANT_xxx */ || \ - defined(MBEDTLS_PSA_CRYPTO_CLIENT) /* The same as the previous, but with separation only */ -#include "mbedtls/config_psa.h" -#endif - -#include "mbedtls/config_adjust_legacy_crypto.h" - -#include "mbedtls/config_adjust_x509.h" - -#include "mbedtls/config_adjust_ssl.h" - -/* Indicate that all configuration symbols are set, - * even the ones that are calculated programmatically. - * It is now safe to query the configuration (to check it, to size buffers, - * etc.). - */ -#define MBEDTLS_CONFIG_IS_FINALIZED - -#include "mbedtls/check_config.h" - -#endif /* MBEDTLS_BUILD_INFO_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/camellia.h b/android/mbedtls/arm64-v8a/include/mbedtls/camellia.h deleted file mode 100644 index 557f47253..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/camellia.h +++ /dev/null @@ -1,305 +0,0 @@ -/** - * \file camellia.h - * - * \brief Camellia block cipher - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_CAMELLIA_H -#define MBEDTLS_CAMELLIA_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -#include "mbedtls/platform_util.h" - -#define MBEDTLS_CAMELLIA_ENCRYPT 1 -#define MBEDTLS_CAMELLIA_DECRYPT 0 - -/** Bad input data. */ -#define MBEDTLS_ERR_CAMELLIA_BAD_INPUT_DATA -0x0024 - -/** Invalid data input length. */ -#define MBEDTLS_ERR_CAMELLIA_INVALID_INPUT_LENGTH -0x0026 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_CAMELLIA_ALT) -// Regular implementation -// - -/** - * \brief CAMELLIA context structure - */ -typedef struct mbedtls_camellia_context { - int MBEDTLS_PRIVATE(nr); /*!< number of rounds */ - uint32_t MBEDTLS_PRIVATE(rk)[68]; /*!< CAMELLIA round keys */ -} -mbedtls_camellia_context; - -#else /* MBEDTLS_CAMELLIA_ALT */ -#include "camellia_alt.h" -#endif /* MBEDTLS_CAMELLIA_ALT */ - -/** - * \brief Initialize a CAMELLIA context. - * - * \param ctx The CAMELLIA context to be initialized. - * This must not be \c NULL. - */ -void mbedtls_camellia_init(mbedtls_camellia_context *ctx); - -/** - * \brief Clear a CAMELLIA context. - * - * \param ctx The CAMELLIA context to be cleared. This may be \c NULL, - * in which case this function returns immediately. If it is not - * \c NULL, it must be initialized. - */ -void mbedtls_camellia_free(mbedtls_camellia_context *ctx); - -/** - * \brief Perform a CAMELLIA key schedule operation for encryption. - * - * \param ctx The CAMELLIA context to use. This must be initialized. - * \param key The encryption key to use. This must be a readable buffer - * of size \p keybits Bits. - * \param keybits The length of \p key in Bits. This must be either \c 128, - * \c 192 or \c 256. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_setkey_enc(mbedtls_camellia_context *ctx, - const unsigned char *key, - unsigned int keybits); - -#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) -/** - * \brief Perform a CAMELLIA key schedule operation for decryption. - * - * \param ctx The CAMELLIA context to use. This must be initialized. - * \param key The decryption key. This must be a readable buffer - * of size \p keybits Bits. - * \param keybits The length of \p key in Bits. This must be either \c 128, - * \c 192 or \c 256. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_setkey_dec(mbedtls_camellia_context *ctx, - const unsigned char *key, - unsigned int keybits); -#endif /* !MBEDTLS_BLOCK_CIPHER_NO_DECRYPT */ - -/** - * \brief Perform a CAMELLIA-ECB block encryption/decryption operation. - * - * \param ctx The CAMELLIA context to use. This must be initialized - * and bound to a key. - * \param mode The mode of operation. This must be either - * #MBEDTLS_CAMELLIA_ENCRYPT or #MBEDTLS_CAMELLIA_DECRYPT. - * \param input The input block. This must be a readable buffer - * of size \c 16 Bytes. - * \param output The output block. This must be a writable buffer - * of size \c 16 Bytes. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_crypt_ecb(mbedtls_camellia_context *ctx, - int mode, - const unsigned char input[16], - unsigned char output[16]); - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -/** - * \brief Perform a CAMELLIA-CBC buffer encryption/decryption operation. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the function same function again on the following - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If on the other hand you need to retain the contents of the - * IV, you should either save it manually or use the cipher - * module instead. - * - * \param ctx The CAMELLIA context to use. This must be initialized - * and bound to a key. - * \param mode The mode of operation. This must be either - * #MBEDTLS_CAMELLIA_ENCRYPT or #MBEDTLS_CAMELLIA_DECRYPT. - * \param length The length in Bytes of the input data \p input. - * This must be a multiple of \c 16 Bytes. - * \param iv The initialization vector. This must be a read/write buffer - * of length \c 16 Bytes. It is updated to allow streaming - * use as explained above. - * \param input The buffer holding the input data. This must point to a - * readable buffer of length \p length Bytes. - * \param output The buffer holding the output data. This must point to a - * writable buffer of length \p length Bytes. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_crypt_cbc(mbedtls_camellia_context *ctx, - int mode, - size_t length, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CBC */ - -#if defined(MBEDTLS_CIPHER_MODE_CFB) -/** - * \brief Perform a CAMELLIA-CFB128 buffer encryption/decryption - * operation. - * - * \note Due to the nature of CFB mode, you should use the same - * key for both encryption and decryption. In particular, calls - * to this function should be preceded by a key-schedule via - * mbedtls_camellia_setkey_enc() regardless of whether \p mode - * is #MBEDTLS_CAMELLIA_ENCRYPT or #MBEDTLS_CAMELLIA_DECRYPT. - * - * \note Upon exit, the content of the IV is updated so that you can - * call the function same function again on the following - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If on the other hand you need to retain the contents of the - * IV, you should either save it manually or use the cipher - * module instead. - * - * \param ctx The CAMELLIA context to use. This must be initialized - * and bound to a key. - * \param mode The mode of operation. This must be either - * #MBEDTLS_CAMELLIA_ENCRYPT or #MBEDTLS_CAMELLIA_DECRYPT. - * \param length The length of the input data \p input. Any value is allowed. - * \param iv_off The current offset in the IV. This must be smaller - * than \c 16 Bytes. It is updated after this call to allow - * the aforementioned streaming usage. - * \param iv The initialization vector. This must be a read/write buffer - * of length \c 16 Bytes. It is updated after this call to - * allow the aforementioned streaming usage. - * \param input The buffer holding the input data. This must be a readable - * buffer of size \p length Bytes. - * \param output The buffer to hold the output data. This must be a writable - * buffer of length \p length Bytes. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_crypt_cfb128(mbedtls_camellia_context *ctx, - int mode, - size_t length, - size_t *iv_off, - unsigned char iv[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CFB */ - -#if defined(MBEDTLS_CIPHER_MODE_CTR) -/** - * \brief Perform a CAMELLIA-CTR buffer encryption/decryption operation. - * - * *note Due to the nature of CTR mode, you should use the same - * key for both encryption and decryption. In particular, calls - * to this function should be preceded by a key-schedule via - * mbedtls_camellia_setkey_enc() regardless of whether the mode - * is #MBEDTLS_CAMELLIA_ENCRYPT or #MBEDTLS_CAMELLIA_DECRYPT. - * - * \warning You must never reuse a nonce value with the same key. Doing so - * would void the encryption for the two messages encrypted with - * the same nonce and key. - * - * There are two common strategies for managing nonces with CTR: - * - * 1. You can handle everything as a single message processed over - * successive calls to this function. In that case, you want to - * set \p nonce_counter and \p nc_off to 0 for the first call, and - * then preserve the values of \p nonce_counter, \p nc_off and \p - * stream_block across calls to this function as they will be - * updated by this function. - * - * With this strategy, you must not encrypt more than 2**128 - * blocks of data with the same key. - * - * 2. You can encrypt separate messages by dividing the \p - * nonce_counter buffer in two areas: the first one used for a - * per-message nonce, handled by yourself, and the second one - * updated by this function internally. - * - * For example, you might reserve the first \c 12 Bytes for the - * per-message nonce, and the last \c 4 Bytes for internal use. - * In that case, before calling this function on a new message you - * need to set the first \c 12 Bytes of \p nonce_counter to your - * chosen nonce value, the last four to \c 0, and \p nc_off to \c 0 - * (which will cause \p stream_block to be ignored). That way, you - * can encrypt at most \c 2**96 messages of up to \c 2**32 blocks - * each with the same key. - * - * The per-message nonce (or information sufficient to reconstruct - * it) needs to be communicated with the ciphertext and must be - * unique. The recommended way to ensure uniqueness is to use a - * message counter. An alternative is to generate random nonces, - * but this limits the number of messages that can be securely - * encrypted: for example, with 96-bit random nonces, you should - * not encrypt more than 2**32 messages with the same key. - * - * Note that for both strategies, sizes are measured in blocks and - * that a CAMELLIA block is \c 16 Bytes. - * - * \warning Upon return, \p stream_block contains sensitive data. Its - * content must not be written to insecure storage and should be - * securely discarded as soon as it's no longer needed. - * - * \param ctx The CAMELLIA context to use. This must be initialized - * and bound to a key. - * \param length The length of the input data \p input in Bytes. - * Any value is allowed. - * \param nc_off The offset in the current \p stream_block (for resuming - * within current cipher stream). The offset pointer to - * should be \c 0 at the start of a stream. It is updated - * at the end of this call. - * \param nonce_counter The 128-bit nonce and counter. This must be a read/write - * buffer of length \c 16 Bytes. - * \param stream_block The saved stream-block for resuming. This must be a - * read/write buffer of length \c 16 Bytes. - * \param input The input data stream. This must be a readable buffer of - * size \p length Bytes. - * \param output The output data stream. This must be a writable buffer - * of size \p length Bytes. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_camellia_crypt_ctr(mbedtls_camellia_context *ctx, - size_t length, - size_t *nc_off, - unsigned char nonce_counter[16], - unsigned char stream_block[16], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CTR */ - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -int mbedtls_camellia_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* camellia.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ccm.h b/android/mbedtls/arm64-v8a/include/mbedtls/ccm.h deleted file mode 100644 index 1da57c921..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ccm.h +++ /dev/null @@ -1,526 +0,0 @@ -/** - * \file ccm.h - * - * \brief This file provides an API for the CCM authenticated encryption - * mode for block ciphers. - * - * CCM combines Counter mode encryption with CBC-MAC authentication - * for 128-bit block ciphers. - * - * Input to CCM includes the following elements: - *
  • Payload - data that is both authenticated and encrypted.
    • - *
  • Associated data (Adata) - data that is authenticated but not - * encrypted, For example, a header.
    • - *
  • Nonce - A unique value that is assigned to the payload and the - * associated data.
- * - * Definition of CCM: - * http://csrc.nist.gov/publications/nistpubs/800-38C/SP800-38C_updated-July20_2007.pdf - * RFC 3610 "Counter with CBC-MAC (CCM)" - * - * Related: - * RFC 5116 "An Interface and Algorithms for Authenticated Encryption" - * - * Definition of CCM*: - * IEEE 802.15.4 - IEEE Standard for Local and metropolitan area networks - * Integer representation is fixed most-significant-octet-first order and - * the representation of octets is most-significant-bit-first order. This is - * consistent with RFC 3610. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CCM_H -#define MBEDTLS_CCM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/cipher.h" - -#if defined(MBEDTLS_BLOCK_CIPHER_C) -#include "mbedtls/block_cipher.h" -#endif - -#define MBEDTLS_CCM_DECRYPT 0 -#define MBEDTLS_CCM_ENCRYPT 1 -#define MBEDTLS_CCM_STAR_DECRYPT 2 -#define MBEDTLS_CCM_STAR_ENCRYPT 3 - -/** Bad input parameters to the function. */ -#define MBEDTLS_ERR_CCM_BAD_INPUT -0x000D -/** Authenticated decryption failed. */ -#define MBEDTLS_ERR_CCM_AUTH_FAILED -0x000F - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_CCM_ALT) -// Regular implementation -// - -/** - * \brief The CCM context-type definition. The CCM context is passed - * to the APIs called. - */ -typedef struct mbedtls_ccm_context { - unsigned char MBEDTLS_PRIVATE(y)[16]; /*!< The Y working buffer */ - unsigned char MBEDTLS_PRIVATE(ctr)[16]; /*!< The counter buffer */ - size_t MBEDTLS_PRIVATE(plaintext_len); /*!< Total plaintext length */ - size_t MBEDTLS_PRIVATE(add_len); /*!< Total authentication data length */ - size_t MBEDTLS_PRIVATE(tag_len); /*!< Total tag length */ - size_t MBEDTLS_PRIVATE(processed); /*!< Track how many bytes of input data - were processed (chunked input). - Used independently for both auth data - and plaintext/ciphertext. - This variable is set to zero after - auth data input is finished. */ - unsigned int MBEDTLS_PRIVATE(q); /*!< The Q working value */ - unsigned int MBEDTLS_PRIVATE(mode); /*!< The operation to perform: - #MBEDTLS_CCM_ENCRYPT or - #MBEDTLS_CCM_DECRYPT or - #MBEDTLS_CCM_STAR_ENCRYPT or - #MBEDTLS_CCM_STAR_DECRYPT. */ -#if defined(MBEDTLS_BLOCK_CIPHER_C) - mbedtls_block_cipher_context_t MBEDTLS_PRIVATE(block_cipher_ctx); /*!< The cipher context used. */ -#else - mbedtls_cipher_context_t MBEDTLS_PRIVATE(cipher_ctx); /*!< The cipher context used. */ -#endif - int MBEDTLS_PRIVATE(state); /*!< Working value holding context's - state. Used for chunked data input */ -} -mbedtls_ccm_context; - -#else /* MBEDTLS_CCM_ALT */ -#include "ccm_alt.h" -#endif /* MBEDTLS_CCM_ALT */ - -/** - * \brief This function initializes the specified CCM context, - * to make references valid, and prepare the context - * for mbedtls_ccm_setkey() or mbedtls_ccm_free(). - * - * \param ctx The CCM context to initialize. This must not be \c NULL. - */ -void mbedtls_ccm_init(mbedtls_ccm_context *ctx); - -/** - * \brief This function initializes the CCM context set in the - * \p ctx parameter and sets the encryption key. - * - * \param ctx The CCM context to initialize. This must be an initialized - * context. - * \param cipher The 128-bit block cipher to use. - * \param key The encryption key. This must not be \c NULL. - * \param keybits The key size in bits. This must be acceptable by the cipher. - * - * \return \c 0 on success. - * \return A CCM or cipher-specific error code on failure. - */ -int mbedtls_ccm_setkey(mbedtls_ccm_context *ctx, - mbedtls_cipher_id_t cipher, - const unsigned char *key, - unsigned int keybits); - -/** - * \brief This function releases and clears the specified CCM context - * and underlying cipher sub-context. - * - * \param ctx The CCM context to clear. If this is \c NULL, the function - * has no effect. Otherwise, this must be initialized. - */ -void mbedtls_ccm_free(mbedtls_ccm_context *ctx); - -/** - * \brief This function encrypts a buffer using CCM. - * - * \note The tag is written to a separate buffer. To concatenate - * the \p tag with the \p output, as done in RFC-3610: - * Counter with CBC-MAC (CCM), use - * \p tag = \p output + \p length, and make sure that the - * output buffer is at least \p length + \p tag_len wide. - * - * \param ctx The CCM context to use for encryption. This must be - * initialized and bound to a key. - * \param length The length of the input data in Bytes. - * \param iv The initialization vector (nonce). This must be a readable - * buffer of at least \p iv_len Bytes. - * \param iv_len The length of the nonce in Bytes: 7, 8, 9, 10, 11, 12, - * or 13. The length L of the message length field is - * 15 - \p iv_len. - * \param ad The additional data field. If \p ad_len is greater than - * zero, \p ad must be a readable buffer of at least that - * length. - * \param ad_len The length of additional data in Bytes. - * This must be less than `2^16 - 2^8`. - * \param input The buffer holding the input data. If \p length is greater - * than zero, \p input must be a readable buffer of at least - * that length. - * \param output The buffer holding the output data. If \p length is greater - * than zero, \p output must be a writable buffer of at least - * that length. - * \param tag The buffer holding the authentication field. This must be a - * writable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the authentication field to generate in Bytes: - * 4, 6, 8, 10, 12, 14 or 16. - * - * \return \c 0 on success. - * \return A CCM or cipher-specific error code on failure. - */ -int mbedtls_ccm_encrypt_and_tag(mbedtls_ccm_context *ctx, size_t length, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, unsigned char *output, - unsigned char *tag, size_t tag_len); - -/** - * \brief This function encrypts a buffer using CCM*. - * - * \note The tag is written to a separate buffer. To concatenate - * the \p tag with the \p output, as done in RFC-3610: - * Counter with CBC-MAC (CCM), use - * \p tag = \p output + \p length, and make sure that the - * output buffer is at least \p length + \p tag_len wide. - * - * \note When using this function in a variable tag length context, - * the tag length has to be encoded into the \p iv passed to - * this function. - * - * \param ctx The CCM context to use for encryption. This must be - * initialized and bound to a key. - * \param length The length of the input data in Bytes. - * For tag length = 0, input length is ignored. - * \param iv The initialization vector (nonce). This must be a readable - * buffer of at least \p iv_len Bytes. - * \param iv_len The length of the nonce in Bytes: 7, 8, 9, 10, 11, 12, - * or 13. The length L of the message length field is - * 15 - \p iv_len. - * \param ad The additional data field. This must be a readable buffer of - * at least \p ad_len Bytes. - * \param ad_len The length of additional data in Bytes. - * This must be less than 2^16 - 2^8. - * \param input The buffer holding the input data. If \p length is greater - * than zero, \p input must be a readable buffer of at least - * that length. - * \param output The buffer holding the output data. If \p length is greater - * than zero, \p output must be a writable buffer of at least - * that length. - * \param tag The buffer holding the authentication field. This must be a - * writable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the authentication field to generate in Bytes: - * 0, 4, 6, 8, 10, 12, 14 or 16. - * - * \warning Passing \c 0 as \p tag_len means that the message is no - * longer authenticated. - * - * \return \c 0 on success. - * \return A CCM or cipher-specific error code on failure. - */ -int mbedtls_ccm_star_encrypt_and_tag(mbedtls_ccm_context *ctx, size_t length, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, unsigned char *output, - unsigned char *tag, size_t tag_len); - -/** - * \brief This function performs a CCM authenticated decryption of a - * buffer. - * - * \param ctx The CCM context to use for decryption. This must be - * initialized and bound to a key. - * \param length The length of the input data in Bytes. - * \param iv The initialization vector (nonce). This must be a readable - * buffer of at least \p iv_len Bytes. - * \param iv_len The length of the nonce in Bytes: 7, 8, 9, 10, 11, 12, - * or 13. The length L of the message length field is - * 15 - \p iv_len. - * \param ad The additional data field. This must be a readable buffer - * of at least that \p ad_len Bytes.. - * \param ad_len The length of additional data in Bytes. - * This must be less than 2^16 - 2^8. - * \param input The buffer holding the input data. If \p length is greater - * than zero, \p input must be a readable buffer of at least - * that length. - * \param output The buffer holding the output data. If \p length is greater - * than zero, \p output must be a writable buffer of at least - * that length. - * \param tag The buffer holding the authentication field. This must be a - * readable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the authentication field to generate in Bytes: - * 4, 6, 8, 10, 12, 14 or 16. - * - * \return \c 0 on success. This indicates that the message is authentic. - * \return #MBEDTLS_ERR_CCM_AUTH_FAILED if the tag does not match. - * \return A cipher-specific error code on calculation failure. - */ -int mbedtls_ccm_auth_decrypt(mbedtls_ccm_context *ctx, size_t length, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, unsigned char *output, - const unsigned char *tag, size_t tag_len); - -/** - * \brief This function performs a CCM* authenticated decryption of a - * buffer. - * - * \note When using this function in a variable tag length context, - * the tag length has to be decoded from \p iv and passed to - * this function as \p tag_len. (\p tag needs to be adjusted - * accordingly.) - * - * \param ctx The CCM context to use for decryption. This must be - * initialized and bound to a key. - * \param length The length of the input data in Bytes. - * For tag length = 0, input length is ignored. - * \param iv The initialization vector (nonce). This must be a readable - * buffer of at least \p iv_len Bytes. - * \param iv_len The length of the nonce in Bytes: 7, 8, 9, 10, 11, 12, - * or 13. The length L of the message length field is - * 15 - \p iv_len. - * \param ad The additional data field. This must be a readable buffer of - * at least that \p ad_len Bytes. - * \param ad_len The length of additional data in Bytes. - * This must be less than 2^16 - 2^8. - * \param input The buffer holding the input data. If \p length is greater - * than zero, \p input must be a readable buffer of at least - * that length. - * \param output The buffer holding the output data. If \p length is greater - * than zero, \p output must be a writable buffer of at least - * that length. - * \param tag The buffer holding the authentication field. This must be a - * readable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the authentication field in Bytes. - * 0, 4, 6, 8, 10, 12, 14 or 16. - * - * \warning Passing \c 0 as \p tag_len means that the message is nos - * longer authenticated. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_AUTH_FAILED if the tag does not match. - * \return A cipher-specific error code on calculation failure. - */ -int mbedtls_ccm_star_auth_decrypt(mbedtls_ccm_context *ctx, size_t length, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, unsigned char *output, - const unsigned char *tag, size_t tag_len); - -/** - * \brief This function starts a CCM encryption or decryption - * operation. - * - * This function and mbedtls_ccm_set_lengths() must be called - * before calling mbedtls_ccm_update_ad() or - * mbedtls_ccm_update(). This function can be called before - * or after mbedtls_ccm_set_lengths(). - * - * \note This function is not implemented in Mbed TLS yet. - * - * \param ctx The CCM context. This must be initialized. - * \param mode The operation to perform: #MBEDTLS_CCM_ENCRYPT or - * #MBEDTLS_CCM_DECRYPT or #MBEDTLS_CCM_STAR_ENCRYPT or - * #MBEDTLS_CCM_STAR_DECRYPT. - * \param iv The initialization vector. This must be a readable buffer - * of at least \p iv_len Bytes. - * \param iv_len The length of the nonce in Bytes: 7, 8, 9, 10, 11, 12, - * or 13. The length L of the message length field is - * 15 - \p iv_len. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_BAD_INPUT on failure: - * \p ctx is in an invalid state, - * \p mode is invalid, - * \p iv_len is invalid (lower than \c 7 or greater than - * \c 13). - */ -int mbedtls_ccm_starts(mbedtls_ccm_context *ctx, - int mode, - const unsigned char *iv, - size_t iv_len); - -/** - * \brief This function declares the lengths of the message - * and additional data for a CCM encryption or decryption - * operation. - * - * This function and mbedtls_ccm_starts() must be called - * before calling mbedtls_ccm_update_ad() or - * mbedtls_ccm_update(). This function can be called before - * or after mbedtls_ccm_starts(). - * - * \note This function is not implemented in Mbed TLS yet. - * - * \param ctx The CCM context. This must be initialized. - * \param total_ad_len The total length of additional data in bytes. - * This must be less than `2^16 - 2^8`. - * \param plaintext_len The length in bytes of the plaintext to encrypt or - * result of the decryption (thus not encompassing the - * additional data that are not encrypted). - * \param tag_len The length of the tag to generate in Bytes: - * 4, 6, 8, 10, 12, 14 or 16. - * For CCM*, zero is also valid. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_BAD_INPUT on failure: - * \p ctx is in an invalid state, - * \p total_ad_len is greater than \c 0xFF00. - */ -int mbedtls_ccm_set_lengths(mbedtls_ccm_context *ctx, - size_t total_ad_len, - size_t plaintext_len, - size_t tag_len); - -/** - * \brief This function feeds an input buffer as associated data - * (authenticated but not encrypted data) in a CCM - * encryption or decryption operation. - * - * You may call this function zero, one or more times - * to pass successive parts of the additional data. The - * lengths \p ad_len of the data parts should eventually add - * up exactly to the total length of additional data - * \c total_ad_len passed to mbedtls_ccm_set_lengths(). You - * may not call this function after calling - * mbedtls_ccm_update(). - * - * \note This function is not implemented in Mbed TLS yet. - * - * \param ctx The CCM context. This must have been started with - * mbedtls_ccm_starts(), the lengths of the message and - * additional data must have been declared with - * mbedtls_ccm_set_lengths() and this must not have yet - * received any input with mbedtls_ccm_update(). - * \param ad The buffer holding the additional data, or \c NULL - * if \p ad_len is \c 0. - * \param ad_len The length of the additional data. If \c 0, - * \p ad may be \c NULL. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_BAD_INPUT on failure: - * \p ctx is in an invalid state, - * total input length too long. - */ -int mbedtls_ccm_update_ad(mbedtls_ccm_context *ctx, - const unsigned char *ad, - size_t ad_len); - -/** - * \brief This function feeds an input buffer into an ongoing CCM - * encryption or decryption operation. - * - * You may call this function zero, one or more times - * to pass successive parts of the input: the plaintext to - * encrypt, or the ciphertext (not including the tag) to - * decrypt. After the last part of the input, call - * mbedtls_ccm_finish(). The lengths \p input_len of the - * data parts should eventually add up exactly to the - * plaintext length \c plaintext_len passed to - * mbedtls_ccm_set_lengths(). - * - * This function may produce output in one of the following - * ways: - * - Immediate output: the output length is always equal - * to the input length. - * - Buffered output: except for the last part of input data, - * the output consists of a whole number of 16-byte blocks. - * If the total input length so far (not including - * associated data) is 16 \* *B* + *A* with *A* < 16 then - * the total output length is 16 \* *B*. - * For the last part of input data, the output length is - * equal to the input length plus the number of bytes (*A*) - * buffered in the previous call to the function (if any). - * The function uses the plaintext length - * \c plaintext_len passed to mbedtls_ccm_set_lengths() - * to detect the last part of input data. - * - * In particular: - * - It is always correct to call this function with - * \p output_size >= \p input_len + 15. - * - If \p input_len is a multiple of 16 for all the calls - * to this function during an operation (not necessary for - * the last one) then it is correct to use \p output_size - * =\p input_len. - * - * \note This function is not implemented in Mbed TLS yet. - * - * \param ctx The CCM context. This must have been started with - * mbedtls_ccm_starts() and the lengths of the message and - * additional data must have been declared with - * mbedtls_ccm_set_lengths(). - * \param input The buffer holding the input data. If \p input_len - * is greater than zero, this must be a readable buffer - * of at least \p input_len bytes. - * \param input_len The length of the input data in bytes. - * \param output The buffer for the output data. If \p output_size - * is greater than zero, this must be a writable buffer of - * at least \p output_size bytes. - * \param output_size The size of the output buffer in bytes. - * See the function description regarding the output size. - * \param output_len On success, \p *output_len contains the actual - * length of the output written in \p output. - * On failure, the content of \p *output_len is - * unspecified. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_BAD_INPUT on failure: - * \p ctx is in an invalid state, - * total input length too long, - * or \p output_size too small. - */ -int mbedtls_ccm_update(mbedtls_ccm_context *ctx, - const unsigned char *input, size_t input_len, - unsigned char *output, size_t output_size, - size_t *output_len); - -/** - * \brief This function finishes the CCM operation and generates - * the authentication tag. - * - * It wraps up the CCM stream, and generates the - * tag. The tag can have a maximum length of 16 Bytes. - * - * \note This function is not implemented in Mbed TLS yet. - * - * \param ctx The CCM context. This must have been started with - * mbedtls_ccm_starts() and the lengths of the message and - * additional data must have been declared with - * mbedtls_ccm_set_lengths(). - * \param tag The buffer for holding the tag. If \p tag_len is greater - * than zero, this must be a writable buffer of at least \p - * tag_len Bytes. - * \param tag_len The length of the tag. Must match the tag length passed to - * mbedtls_ccm_set_lengths() function. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CCM_BAD_INPUT on failure: - * \p ctx is in an invalid state, - * invalid value of \p tag_len, - * the total amount of additional data passed to - * mbedtls_ccm_update_ad() was lower than the total length of - * additional data \c total_ad_len passed to - * mbedtls_ccm_set_lengths(), - * the total amount of input data passed to - * mbedtls_ccm_update() was lower than the plaintext length - * \c plaintext_len passed to mbedtls_ccm_set_lengths(). - */ -int mbedtls_ccm_finish(mbedtls_ccm_context *ctx, - unsigned char *tag, size_t tag_len); - -#if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_CCM_GCM_CAN_AES) -/** - * \brief The CCM checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_ccm_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST && MBEDTLS_AES_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_CCM_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/chacha20.h b/android/mbedtls/arm64-v8a/include/mbedtls/chacha20.h deleted file mode 100644 index 680fe3604..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/chacha20.h +++ /dev/null @@ -1,202 +0,0 @@ -/** - * \file chacha20.h - * - * \brief This file contains ChaCha20 definitions and functions. - * - * ChaCha20 is a stream cipher that can encrypt and decrypt - * information. ChaCha was created by Daniel Bernstein as a variant of - * its Salsa cipher https://cr.yp.to/chacha/chacha-20080128.pdf - * ChaCha20 is the variant with 20 rounds, that was also standardized - * in RFC 7539. - * - * \author Daniel King - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CHACHA20_H -#define MBEDTLS_CHACHA20_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -/** Invalid input parameter(s). */ -#define MBEDTLS_ERR_CHACHA20_BAD_INPUT_DATA -0x0051 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_CHACHA20_ALT) - -typedef struct mbedtls_chacha20_context { - uint32_t MBEDTLS_PRIVATE(state)[16]; /*! The state (before round operations). */ - uint8_t MBEDTLS_PRIVATE(keystream8)[64]; /*! Leftover keystream bytes. */ - size_t MBEDTLS_PRIVATE(keystream_bytes_used); /*! Number of keystream bytes already used. */ -} -mbedtls_chacha20_context; - -#else /* MBEDTLS_CHACHA20_ALT */ -#include "chacha20_alt.h" -#endif /* MBEDTLS_CHACHA20_ALT */ - -/** - * \brief This function initializes the specified ChaCha20 context. - * - * It must be the first API called before using - * the context. - * - * It is usually followed by calls to - * \c mbedtls_chacha20_setkey() and - * \c mbedtls_chacha20_starts(), then one or more calls to - * to \c mbedtls_chacha20_update(), and finally to - * \c mbedtls_chacha20_free(). - * - * \param ctx The ChaCha20 context to initialize. - * This must not be \c NULL. - */ -void mbedtls_chacha20_init(mbedtls_chacha20_context *ctx); - -/** - * \brief This function releases and clears the specified - * ChaCha20 context. - * - * \param ctx The ChaCha20 context to clear. This may be \c NULL, - * in which case this function is a no-op. If it is not - * \c NULL, it must point to an initialized context. - * - */ -void mbedtls_chacha20_free(mbedtls_chacha20_context *ctx); - -/** - * \brief This function sets the encryption/decryption key. - * - * \note After using this function, you must also call - * \c mbedtls_chacha20_starts() to set a nonce before you - * start encrypting/decrypting data with - * \c mbedtls_chacha_update(). - * - * \param ctx The ChaCha20 context to which the key should be bound. - * It must be initialized. - * \param key The encryption/decryption key. This must be \c 32 Bytes - * in length. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CHACHA20_BAD_INPUT_DATA if ctx or key is NULL. - */ -int mbedtls_chacha20_setkey(mbedtls_chacha20_context *ctx, - const unsigned char key[32]); - -/** - * \brief This function sets the nonce and initial counter value. - * - * \note A ChaCha20 context can be re-used with the same key by - * calling this function to change the nonce. - * - * \warning You must never use the same nonce twice with the same key. - * This would void any confidentiality guarantees for the - * messages encrypted with the same nonce and key. - * - * \param ctx The ChaCha20 context to which the nonce should be bound. - * It must be initialized and bound to a key. - * \param nonce The nonce. This must be \c 12 Bytes in size. - * \param counter The initial counter value. This is usually \c 0. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CHACHA20_BAD_INPUT_DATA if ctx or nonce is - * NULL. - */ -int mbedtls_chacha20_starts(mbedtls_chacha20_context *ctx, - const unsigned char nonce[12], - uint32_t counter); - -/** - * \brief This function encrypts or decrypts data. - * - * Since ChaCha20 is a stream cipher, the same operation is - * used for encrypting and decrypting data. - * - * \note The \p input and \p output pointers must either be equal or - * point to non-overlapping buffers. - * - * \note \c mbedtls_chacha20_setkey() and - * \c mbedtls_chacha20_starts() must be called at least once - * to setup the context before this function can be called. - * - * \note This function can be called multiple times in a row in - * order to encrypt of decrypt data piecewise with the same - * key and nonce. - * - * \param ctx The ChaCha20 context to use for encryption or decryption. - * It must be initialized and bound to a key and nonce. - * \param size The length of the input data in Bytes. - * \param input The buffer holding the input data. - * This pointer can be \c NULL if `size == 0`. - * \param output The buffer holding the output data. - * This must be able to hold \p size Bytes. - * This pointer can be \c NULL if `size == 0`. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_chacha20_update(mbedtls_chacha20_context *ctx, - size_t size, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function encrypts or decrypts data with ChaCha20 and - * the given key and nonce. - * - * Since ChaCha20 is a stream cipher, the same operation is - * used for encrypting and decrypting data. - * - * \warning You must never use the same (key, nonce) pair more than - * once. This would void any confidentiality guarantees for - * the messages encrypted with the same nonce and key. - * - * \note The \p input and \p output pointers must either be equal or - * point to non-overlapping buffers. - * - * \param key The encryption/decryption key. - * This must be \c 32 Bytes in length. - * \param nonce The nonce. This must be \c 12 Bytes in size. - * \param counter The initial counter value. This is usually \c 0. - * \param size The length of the input data in Bytes. - * \param input The buffer holding the input data. - * This pointer can be \c NULL if `size == 0`. - * \param output The buffer holding the output data. - * This must be able to hold \p size Bytes. - * This pointer can be \c NULL if `size == 0`. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_chacha20_crypt(const unsigned char key[32], - const unsigned char nonce[12], - uint32_t counter, - size_t size, - const unsigned char *input, - unsigned char *output); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief The ChaCha20 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_chacha20_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_CHACHA20_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/chachapoly.h b/android/mbedtls/arm64-v8a/include/mbedtls/chachapoly.h deleted file mode 100644 index 3dc21e380..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/chachapoly.h +++ /dev/null @@ -1,342 +0,0 @@ -/** - * \file chachapoly.h - * - * \brief This file contains the AEAD-ChaCha20-Poly1305 definitions and - * functions. - * - * ChaCha20-Poly1305 is an algorithm for Authenticated Encryption - * with Associated Data (AEAD) that can be used to encrypt and - * authenticate data. It is based on ChaCha20 and Poly1305 by Daniel - * Bernstein and was standardized in RFC 7539. - * - * \author Daniel King - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CHACHAPOLY_H -#define MBEDTLS_CHACHAPOLY_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -/* for shared error codes */ -#include "mbedtls/poly1305.h" - -/** The requested operation is not permitted in the current state. */ -#define MBEDTLS_ERR_CHACHAPOLY_BAD_STATE -0x0054 -/** Authenticated decryption failed: data was not authentic. */ -#define MBEDTLS_ERR_CHACHAPOLY_AUTH_FAILED -0x0056 - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - MBEDTLS_CHACHAPOLY_ENCRYPT, /**< The mode value for performing encryption. */ - MBEDTLS_CHACHAPOLY_DECRYPT /**< The mode value for performing decryption. */ -} -mbedtls_chachapoly_mode_t; - -#if !defined(MBEDTLS_CHACHAPOLY_ALT) - -#include "mbedtls/chacha20.h" - -typedef struct mbedtls_chachapoly_context { - mbedtls_chacha20_context MBEDTLS_PRIVATE(chacha20_ctx); /**< The ChaCha20 context. */ - mbedtls_poly1305_context MBEDTLS_PRIVATE(poly1305_ctx); /**< The Poly1305 context. */ - uint64_t MBEDTLS_PRIVATE(aad_len); /**< The length (bytes) of the Additional Authenticated Data. */ - uint64_t MBEDTLS_PRIVATE(ciphertext_len); /**< The length (bytes) of the ciphertext. */ - int MBEDTLS_PRIVATE(state); /**< The current state of the context. */ - mbedtls_chachapoly_mode_t MBEDTLS_PRIVATE(mode); /**< Cipher mode (encrypt or decrypt). */ -} -mbedtls_chachapoly_context; - -#else /* !MBEDTLS_CHACHAPOLY_ALT */ -#include "chachapoly_alt.h" -#endif /* !MBEDTLS_CHACHAPOLY_ALT */ - -/** - * \brief This function initializes the specified ChaCha20-Poly1305 context. - * - * It must be the first API called before using - * the context. It must be followed by a call to - * \c mbedtls_chachapoly_setkey() before any operation can be - * done, and to \c mbedtls_chachapoly_free() once all - * operations with that context have been finished. - * - * In order to encrypt or decrypt full messages at once, for - * each message you should make a single call to - * \c mbedtls_chachapoly_crypt_and_tag() or - * \c mbedtls_chachapoly_auth_decrypt(). - * - * In order to encrypt messages piecewise, for each - * message you should make a call to - * \c mbedtls_chachapoly_starts(), then 0 or more calls to - * \c mbedtls_chachapoly_update_aad(), then 0 or more calls to - * \c mbedtls_chachapoly_update(), then one call to - * \c mbedtls_chachapoly_finish(). - * - * \warning Decryption with the piecewise API is discouraged! Always - * use \c mbedtls_chachapoly_auth_decrypt() when possible! - * - * If however this is not possible because the data is too - * large to fit in memory, you need to: - * - * - call \c mbedtls_chachapoly_starts() and (if needed) - * \c mbedtls_chachapoly_update_aad() as above, - * - call \c mbedtls_chachapoly_update() multiple times and - * ensure its output (the plaintext) is NOT used in any other - * way than placing it in temporary storage at this point, - * - call \c mbedtls_chachapoly_finish() to compute the - * authentication tag and compared it in constant time to the - * tag received with the ciphertext. - * - * If the tags are not equal, you must immediately discard - * all previous outputs of \c mbedtls_chachapoly_update(), - * otherwise you can now safely use the plaintext. - * - * \param ctx The ChachaPoly context to initialize. Must not be \c NULL. - */ -void mbedtls_chachapoly_init(mbedtls_chachapoly_context *ctx); - -/** - * \brief This function releases and clears the specified - * ChaCha20-Poly1305 context. - * - * \param ctx The ChachaPoly context to clear. This may be \c NULL, in which - * case this function is a no-op. - */ -void mbedtls_chachapoly_free(mbedtls_chachapoly_context *ctx); - -/** - * \brief This function sets the ChaCha20-Poly1305 - * symmetric encryption key. - * - * \param ctx The ChaCha20-Poly1305 context to which the key should be - * bound. This must be initialized. - * \param key The \c 256 Bit (\c 32 Bytes) key. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_chachapoly_setkey(mbedtls_chachapoly_context *ctx, - const unsigned char key[32]); - -/** - * \brief This function starts a ChaCha20-Poly1305 encryption or - * decryption operation. - * - * \warning You must never use the same nonce twice with the same key. - * This would void any confidentiality and authenticity - * guarantees for the messages encrypted with the same nonce - * and key. - * - * \note If the context is being used for AAD only (no data to - * encrypt or decrypt) then \p mode can be set to any value. - * - * \warning Decryption with the piecewise API is discouraged, see the - * warning on \c mbedtls_chachapoly_init(). - * - * \param ctx The ChaCha20-Poly1305 context. This must be initialized - * and bound to a key. - * \param nonce The nonce/IV to use for the message. - * This must be a readable buffer of length \c 12 Bytes. - * \param mode The operation to perform: #MBEDTLS_CHACHAPOLY_ENCRYPT or - * #MBEDTLS_CHACHAPOLY_DECRYPT (discouraged, see warning). - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_chachapoly_starts(mbedtls_chachapoly_context *ctx, - const unsigned char nonce[12], - mbedtls_chachapoly_mode_t mode); - -/** - * \brief This function feeds additional data to be authenticated - * into an ongoing ChaCha20-Poly1305 operation. - * - * The Additional Authenticated Data (AAD), also called - * Associated Data (AD) is only authenticated but not - * encrypted nor included in the encrypted output. It is - * usually transmitted separately from the ciphertext or - * computed locally by each party. - * - * \note This function is called before data is encrypted/decrypted. - * I.e. call this function to process the AAD before calling - * \c mbedtls_chachapoly_update(). - * - * You may call this function multiple times to process - * an arbitrary amount of AAD. It is permitted to call - * this function 0 times, if no AAD is used. - * - * This function cannot be called any more if data has - * been processed by \c mbedtls_chachapoly_update(), - * or if the context has been finished. - * - * \warning Decryption with the piecewise API is discouraged, see the - * warning on \c mbedtls_chachapoly_init(). - * - * \param ctx The ChaCha20-Poly1305 context. This must be initialized - * and bound to a key. - * \param aad_len The length in Bytes of the AAD. The length has no - * restrictions. - * \param aad Buffer containing the AAD. - * This pointer can be \c NULL if `aad_len == 0`. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_POLY1305_BAD_INPUT_DATA - * if \p ctx or \p aad are NULL. - * \return #MBEDTLS_ERR_CHACHAPOLY_BAD_STATE - * if the operations has not been started or has been - * finished, or if the AAD has been finished. - */ -int mbedtls_chachapoly_update_aad(mbedtls_chachapoly_context *ctx, - const unsigned char *aad, - size_t aad_len); - -/** - * \brief Thus function feeds data to be encrypted or decrypted - * into an on-going ChaCha20-Poly1305 - * operation. - * - * The direction (encryption or decryption) depends on the - * mode that was given when calling - * \c mbedtls_chachapoly_starts(). - * - * You may call this function multiple times to process - * an arbitrary amount of data. It is permitted to call - * this function 0 times, if no data is to be encrypted - * or decrypted. - * - * \warning Decryption with the piecewise API is discouraged, see the - * warning on \c mbedtls_chachapoly_init(). - * - * \param ctx The ChaCha20-Poly1305 context to use. This must be initialized. - * \param len The length (in bytes) of the data to encrypt or decrypt. - * \param input The buffer containing the data to encrypt or decrypt. - * This pointer can be \c NULL if `len == 0`. - * \param output The buffer to where the encrypted or decrypted data is - * written. This must be able to hold \p len bytes. - * This pointer can be \c NULL if `len == 0`. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CHACHAPOLY_BAD_STATE - * if the operation has not been started or has been - * finished. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_chachapoly_update(mbedtls_chachapoly_context *ctx, - size_t len, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function finished the ChaCha20-Poly1305 operation and - * generates the MAC (authentication tag). - * - * \param ctx The ChaCha20-Poly1305 context to use. This must be initialized. - * \param mac The buffer to where the 128-bit (16 bytes) MAC is written. - * - * \warning Decryption with the piecewise API is discouraged, see the - * warning on \c mbedtls_chachapoly_init(). - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CHACHAPOLY_BAD_STATE - * if the operation has not been started or has been - * finished. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_chachapoly_finish(mbedtls_chachapoly_context *ctx, - unsigned char mac[16]); - -/** - * \brief This function performs a complete ChaCha20-Poly1305 - * authenticated encryption with the previously-set key. - * - * \note Before using this function, you must set the key with - * \c mbedtls_chachapoly_setkey(). - * - * \warning You must never use the same nonce twice with the same key. - * This would void any confidentiality and authenticity - * guarantees for the messages encrypted with the same nonce - * and key. - * - * \param ctx The ChaCha20-Poly1305 context to use (holds the key). - * This must be initialized. - * \param length The length (in bytes) of the data to encrypt or decrypt. - * \param nonce The 96-bit (12 bytes) nonce/IV to use. - * \param aad The buffer containing the additional authenticated - * data (AAD). This pointer can be \c NULL if `aad_len == 0`. - * \param aad_len The length (in bytes) of the AAD data to process. - * \param input The buffer containing the data to encrypt or decrypt. - * This pointer can be \c NULL if `ilen == 0`. - * \param output The buffer to where the encrypted or decrypted data - * is written. This pointer can be \c NULL if `ilen == 0`. - * \param tag The buffer to where the computed 128-bit (16 bytes) MAC - * is written. This must not be \c NULL. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_chachapoly_encrypt_and_tag(mbedtls_chachapoly_context *ctx, - size_t length, - const unsigned char nonce[12], - const unsigned char *aad, - size_t aad_len, - const unsigned char *input, - unsigned char *output, - unsigned char tag[16]); - -/** - * \brief This function performs a complete ChaCha20-Poly1305 - * authenticated decryption with the previously-set key. - * - * \note Before using this function, you must set the key with - * \c mbedtls_chachapoly_setkey(). - * - * \param ctx The ChaCha20-Poly1305 context to use (holds the key). - * \param length The length (in Bytes) of the data to decrypt. - * \param nonce The \c 96 Bit (\c 12 bytes) nonce/IV to use. - * \param aad The buffer containing the additional authenticated data (AAD). - * This pointer can be \c NULL if `aad_len == 0`. - * \param aad_len The length (in bytes) of the AAD data to process. - * \param tag The buffer holding the authentication tag. - * This must be a readable buffer of length \c 16 Bytes. - * \param input The buffer containing the data to decrypt. - * This pointer can be \c NULL if `ilen == 0`. - * \param output The buffer to where the decrypted data is written. - * This pointer can be \c NULL if `ilen == 0`. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CHACHAPOLY_AUTH_FAILED - * if the data was not authentic. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_chachapoly_auth_decrypt(mbedtls_chachapoly_context *ctx, - size_t length, - const unsigned char nonce[12], - const unsigned char *aad, - size_t aad_len, - const unsigned char tag[16], - const unsigned char *input, - unsigned char *output); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief The ChaCha20-Poly1305 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_chachapoly_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_CHACHAPOLY_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/check_config.h b/android/mbedtls/arm64-v8a/include/mbedtls/check_config.h deleted file mode 100644 index aec5050b7..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/check_config.h +++ /dev/null @@ -1,1149 +0,0 @@ -/** - * \file check_config.h - * - * \brief Consistency checks for configuration options - * - * This is an internal header. Do not include it directly. - * - * This header is included automatically by all public Mbed TLS headers - * (via mbedtls/build_info.h). Do not include it directly in a configuration - * file such as mbedtls/mbedtls_config.h or #MBEDTLS_USER_CONFIG_FILE! - * It would run at the wrong time due to missing derived symbols. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CHECK_CONFIG_H -#define MBEDTLS_CHECK_CONFIG_H - -/* *INDENT-OFF* */ - -#if !defined(MBEDTLS_CONFIG_IS_FINALIZED) -#warning "Do not include mbedtls/check_config.h manually! " \ - "This may cause spurious errors. " \ - "It is included automatically at the right point since Mbed TLS 3.0." -#endif /* !MBEDTLS_CONFIG_IS_FINALIZED */ - -/* - * We assume CHAR_BIT is 8 in many places. In practice, this is true on our - * target platforms, so not an issue, but let's just be extra sure. - */ -#include -#if CHAR_BIT != 8 -#error "Mbed TLS requires a platform with 8-bit chars" -#endif - -#include - -#if defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER <= 1900) -#if !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_C is required on Windows" -#endif -/* See auto-enabling SNPRINTF_ALT and VSNPRINTF_ALT - * in * config_adjust_legacy_crypto.h */ -#endif /* _MINGW32__ || (_MSC_VER && (_MSC_VER <= 1900)) */ - -#if defined(TARGET_LIKE_MBED) && defined(MBEDTLS_NET_C) -#error "The NET module is not available for mbed OS - please use the network functions provided by Mbed OS" -#endif - -#if defined(MBEDTLS_DEPRECATED_WARNING) && \ - !defined(__GNUC__) && !defined(__clang__) -#error "MBEDTLS_DEPRECATED_WARNING only works with GCC and Clang" -#endif - -#if defined(MBEDTLS_HAVE_TIME_DATE) && !defined(MBEDTLS_HAVE_TIME) -#error "MBEDTLS_HAVE_TIME_DATE without MBEDTLS_HAVE_TIME does not make sense" -#endif - -/* Limitations on ECC key types acceleration: if we have any of `PUBLIC_KEY`, - * `KEY_PAIR_BASIC`, `KEY_PAIR_IMPORT`, `KEY_PAIR_EXPORT` then we must have - * all 4 of them. - */ -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY) || \ - defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ - defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ - defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY) || \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT) -#error "Unsupported partial support for ECC key type acceleration, see docs/driver-only-builds.md" -#endif /* not all of public, basic, import, export */ -#endif /* one of public, basic, import, export */ - -/* Limitations on ECC curves acceleration: partial curve acceleration is only - * supported with crypto excluding PK, X.509 or TLS. - * Note: no need to check X.509 as it depends on PK. */ -#if defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_256) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_384) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_512) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_255) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_448) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_192) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_224) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_256) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_192) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_224) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_256) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_384) || \ - defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_521) -#if defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) -#if defined(MBEDTLS_PK_C) || \ - defined(MBEDTLS_SSL_TLS_C) -#error "Unsupported partial support for ECC curves acceleration, see docs/driver-only-builds.md" -#endif /* modules beyond what's supported */ -#endif /* not all curves accelerated */ -#endif /* some curve accelerated */ - -#if defined(MBEDTLS_CTR_DRBG_C) && !(defined(MBEDTLS_AES_C) || \ - (defined(MBEDTLS_PSA_CRYPTO_CLIENT) && defined(PSA_WANT_KEY_TYPE_AES) && \ - defined(PSA_WANT_ALG_ECB_NO_PADDING))) -#error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_DHM_C) && !defined(MBEDTLS_BIGNUM_C) -#error "MBEDTLS_DHM_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_CMAC_C) && \ - ( !defined(MBEDTLS_CIPHER_C ) || ( !defined(MBEDTLS_AES_C) && !defined(MBEDTLS_DES_C) ) ) -#error "MBEDTLS_CMAC_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_NIST_KW_C) && \ - ( !defined(MBEDTLS_AES_C) || !defined(MBEDTLS_CIPHER_C) ) -#error "MBEDTLS_NIST_KW_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) && defined(MBEDTLS_PSA_CRYPTO_CONFIG) -#if defined(PSA_WANT_ALG_CBC_NO_PADDING) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and PSA_WANT_ALG_CBC_NO_PADDING cannot be defined simultaneously" -#endif -#if defined(PSA_WANT_ALG_CBC_PKCS7) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and PSA_WANT_ALG_CBC_PKCS7 cannot be defined simultaneously" -#endif -#if defined(PSA_WANT_ALG_ECB_NO_PADDING) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and PSA_WANT_ALG_ECB_NO_PADDING cannot be defined simultaneously" -#endif -#if defined(PSA_WANT_KEY_TYPE_DES) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and PSA_WANT_KEY_TYPE_DES cannot be defined simultaneously" -#endif -#endif - -#if defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT) -#if defined(MBEDTLS_CIPHER_MODE_CBC) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and MBEDTLS_CIPHER_MODE_CBC cannot be defined simultaneously" -#endif -#if defined(MBEDTLS_CIPHER_MODE_XTS) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and MBEDTLS_CIPHER_MODE_XTS cannot be defined simultaneously" -#endif -#if defined(MBEDTLS_DES_C) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and MBEDTLS_DES_C cannot be defined simultaneously" -#endif -#if defined(MBEDTLS_NIST_KW_C) -#error "MBEDTLS_BLOCK_CIPHER_NO_DECRYPT and MBEDTLS_NIST_KW_C cannot be defined simultaneously" -#endif -#endif - -#if defined(MBEDTLS_ECDH_C) && !defined(MBEDTLS_ECP_C) -#error "MBEDTLS_ECDH_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECDSA_C) && \ - ( !defined(MBEDTLS_ECP_C) || \ - !( defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) ) || \ - !defined(MBEDTLS_ASN1_PARSE_C) || \ - !defined(MBEDTLS_ASN1_WRITE_C) ) -#error "MBEDTLS_ECDSA_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PK_C) && defined(MBEDTLS_USE_PSA_CRYPTO) -#if defined(MBEDTLS_PK_CAN_ECDSA_SIGN) && !defined(MBEDTLS_ASN1_WRITE_C) -#error "MBEDTLS_PK_C with MBEDTLS_USE_PSA_CRYPTO needs MBEDTLS_ASN1_WRITE_C for ECDSA signature" -#endif -#if defined(MBEDTLS_PK_CAN_ECDSA_VERIFY) && !defined(MBEDTLS_ASN1_PARSE_C) -#error "MBEDTLS_PK_C with MBEDTLS_USE_PSA_CRYPTO needs MBEDTLS_ASN1_PARSE_C for ECDSA verification" -#endif -#endif /* MBEDTLS_PK_C && MBEDTLS_USE_PSA_CRYPTO */ - -#if defined(MBEDTLS_ECJPAKE_C) && \ - !defined(MBEDTLS_ECP_C) -#error "MBEDTLS_ECJPAKE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_RESTARTABLE) && \ - ( defined(MBEDTLS_ECDH_COMPUTE_SHARED_ALT) || \ - defined(MBEDTLS_ECDH_GEN_PUBLIC_ALT) || \ - defined(MBEDTLS_ECDSA_SIGN_ALT) || \ - defined(MBEDTLS_ECDSA_VERIFY_ALT) || \ - defined(MBEDTLS_ECDSA_GENKEY_ALT) || \ - defined(MBEDTLS_ECP_INTERNAL_ALT) || \ - defined(MBEDTLS_ECP_ALT) ) -#error "MBEDTLS_ECP_RESTARTABLE defined, but it cannot coexist with an alternative ECP implementation" -#endif - -#if defined(MBEDTLS_ECP_RESTARTABLE) && \ - !defined(MBEDTLS_ECP_C) -#error "MBEDTLS_ECP_RESTARTABLE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) && !defined(MBEDTLS_HMAC_DRBG_C) -#error "MBEDTLS_ECDSA_DETERMINISTIC defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_LIGHT) && ( !defined(MBEDTLS_BIGNUM_C) || ( \ - !defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) && \ - !defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) ) ) -#error "MBEDTLS_ECP_C defined (or a subset enabled), but not all prerequisites" -#endif - -#if defined(MBEDTLS_ENTROPY_C) && \ - !(defined(MBEDTLS_MD_CAN_SHA512) || defined(MBEDTLS_MD_CAN_SHA256)) -#error "MBEDTLS_ENTROPY_C defined, but not all prerequisites" -#endif -#if defined(MBEDTLS_ENTROPY_C) && \ - defined(MBEDTLS_CTR_DRBG_ENTROPY_LEN) && (MBEDTLS_CTR_DRBG_ENTROPY_LEN > 64) -#error "MBEDTLS_CTR_DRBG_ENTROPY_LEN value too high" -#endif -#if defined(MBEDTLS_ENTROPY_C) && \ - (defined(MBEDTLS_ENTROPY_FORCE_SHA256) || !defined(MBEDTLS_MD_CAN_SHA512)) \ - && defined(MBEDTLS_CTR_DRBG_ENTROPY_LEN) && (MBEDTLS_CTR_DRBG_ENTROPY_LEN > 32) -#error "MBEDTLS_CTR_DRBG_ENTROPY_LEN value too high" -#endif -#if defined(MBEDTLS_ENTROPY_C) && \ - defined(MBEDTLS_ENTROPY_FORCE_SHA256) && !defined(MBEDTLS_MD_CAN_SHA256) -#error "MBEDTLS_ENTROPY_FORCE_SHA256 defined, but not all prerequisites" -#endif - -#if defined(__has_feature) -#if __has_feature(memory_sanitizer) -#define MBEDTLS_HAS_MEMSAN // #undef at the end of this paragraph -#endif -#endif -#if defined(MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN) && !defined(MBEDTLS_HAS_MEMSAN) -#error "MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN requires building with MemorySanitizer" -#endif -#if defined(MBEDTLS_HAS_MEMSAN) && defined(MBEDTLS_HAVE_ASM) -#error "MemorySanitizer does not support assembly implementation" -#endif -#undef MBEDTLS_HAS_MEMSAN // temporary macro defined above - -#if defined(MBEDTLS_CCM_C) && \ - !(defined(MBEDTLS_CCM_GCM_CAN_AES) || defined(MBEDTLS_CCM_GCM_CAN_ARIA) || \ - defined(MBEDTLS_CCM_GCM_CAN_CAMELLIA)) -#error "MBEDTLS_CCM_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_GCM_C) && \ - !(defined(MBEDTLS_CCM_GCM_CAN_AES) || defined(MBEDTLS_CCM_GCM_CAN_ARIA) || \ - defined(MBEDTLS_CCM_GCM_CAN_CAMELLIA)) -#error "MBEDTLS_GCM_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_CHACHAPOLY_C) && !defined(MBEDTLS_CHACHA20_C) -#error "MBEDTLS_CHACHAPOLY_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_CHACHAPOLY_C) && !defined(MBEDTLS_POLY1305_C) -#error "MBEDTLS_CHACHAPOLY_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_RANDOMIZE_JAC_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_RANDOMIZE_JAC_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_ADD_MIXED_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_ADD_MIXED_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_DOUBLE_JAC_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_DOUBLE_JAC_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_NORMALIZE_JAC_MANY_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_NORMALIZE_JAC_MANY_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_NORMALIZE_JAC_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_NORMALIZE_JAC_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_DOUBLE_ADD_MXZ_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_DOUBLE_ADD_MXZ_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_RANDOMIZE_MXZ_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_RANDOMIZE_MXZ_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_NORMALIZE_MXZ_ALT) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_NORMALIZE_MXZ_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ECP_NO_FALLBACK) && !defined(MBEDTLS_ECP_INTERNAL_ALT) -#error "MBEDTLS_ECP_NO_FALLBACK defined, but no alternative implementation enabled" -#endif - -#if defined(MBEDTLS_HKDF_C) && !defined(MBEDTLS_MD_C) -#error "MBEDTLS_HKDF_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_HMAC_DRBG_C) && !defined(MBEDTLS_MD_C) -#error "MBEDTLS_HMAC_DRBG_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) && \ - ( !defined(MBEDTLS_CAN_ECDH) || \ - !defined(MBEDTLS_PK_CAN_ECDSA_SIGN) || \ - !defined(MBEDTLS_X509_CRT_PARSE_C) ) -#error "MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) && \ - ( !defined(MBEDTLS_CAN_ECDH) || !defined(MBEDTLS_RSA_C) || \ - !defined(MBEDTLS_X509_CRT_PARSE_C) ) -#error "MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) && !defined(MBEDTLS_DHM_C) -#error "MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) && \ - !defined(MBEDTLS_CAN_ECDH) -#error "MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) && \ - ( !defined(MBEDTLS_DHM_C) || !defined(MBEDTLS_RSA_C) || \ - !defined(MBEDTLS_X509_CRT_PARSE_C) || !defined(MBEDTLS_PKCS1_V15) ) -#error "MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) && \ - ( !defined(MBEDTLS_CAN_ECDH) || !defined(MBEDTLS_RSA_C) || \ - !defined(MBEDTLS_X509_CRT_PARSE_C) || !defined(MBEDTLS_PKCS1_V15) ) -#error "MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) && \ - ( !defined(MBEDTLS_CAN_ECDH) || \ - !defined(MBEDTLS_PK_CAN_ECDSA_SIGN) || \ - !defined(MBEDTLS_X509_CRT_PARSE_C) ) -#error "MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) && \ - ( !defined(MBEDTLS_RSA_C) || !defined(MBEDTLS_X509_CRT_PARSE_C) || \ - !defined(MBEDTLS_PKCS1_V15) ) -#error "MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) && \ - ( !defined(MBEDTLS_RSA_C) || !defined(MBEDTLS_X509_CRT_PARSE_C) || \ - !defined(MBEDTLS_PKCS1_V15) ) -#error "MBEDTLS_KEY_EXCHANGE_RSA_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -#if defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) && \ - ( !defined(PSA_WANT_ALG_JPAKE) || \ - !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ - !defined(PSA_WANT_ECC_SECP_R1_256) ) -#error "MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED defined, but not all prerequisites" -#endif -#else /* MBEDTLS_USE_PSA_CRYPTO */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) && \ - ( !defined(MBEDTLS_ECJPAKE_C) || \ - !defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) ) -#error "MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED defined, but not all prerequisites" -#endif -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -/* Use of EC J-PAKE in TLS requires SHA-256. */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) && \ - !defined(MBEDTLS_MD_CAN_SHA256) -#error "MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED) && \ - !defined(MBEDTLS_SSL_KEEP_PEER_CERTIFICATE) && \ - !defined(MBEDTLS_MD_CAN_SHA256) && \ - !defined(MBEDTLS_MD_CAN_SHA512) && \ - !defined(MBEDTLS_MD_CAN_SHA1) -#error "!MBEDTLS_SSL_KEEP_PEER_CERTIFICATE requires SHA-512, SHA-256 or SHA-1". -#endif - -#if defined(MBEDTLS_MD_C) && \ - !defined(MBEDTLS_MD_CAN_MD5) && \ - !defined(MBEDTLS_MD_CAN_RIPEMD160) && \ - !defined(MBEDTLS_MD_CAN_SHA1) && \ - !defined(MBEDTLS_MD_CAN_SHA224) && \ - !defined(MBEDTLS_MD_CAN_SHA256) && \ - !defined(MBEDTLS_MD_CAN_SHA384) && \ - !defined(MBEDTLS_MD_CAN_SHA512) && \ - !defined(MBEDTLS_MD_CAN_SHA3_224) && \ - !defined(MBEDTLS_MD_CAN_SHA3_256) && \ - !defined(MBEDTLS_MD_CAN_SHA3_384) && \ - !defined(MBEDTLS_MD_CAN_SHA3_512) -#error "MBEDTLS_MD_C defined, but no hash algorithm" -#endif - -#if defined(MBEDTLS_LMS_C) && \ - ! ( defined(MBEDTLS_PSA_CRYPTO_CLIENT) && defined(PSA_WANT_ALG_SHA_256) ) -#error "MBEDTLS_LMS_C requires MBEDTLS_PSA_CRYPTO_C and PSA_WANT_ALG_SHA_256" -#endif - -#if defined(MBEDTLS_LMS_PRIVATE) && \ - ( !defined(MBEDTLS_LMS_C) ) -#error "MBEDTLS_LMS_PRIVATE requires MBEDTLS_LMS_C" -#endif - -#if defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) && \ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_PLATFORM_MEMORY) ) -#error "MBEDTLS_MEMORY_BUFFER_ALLOC_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_MEMORY_BACKTRACE) && !defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) -#error "MBEDTLS_MEMORY_BACKTRACE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_MEMORY_DEBUG) && !defined(MBEDTLS_MEMORY_BUFFER_ALLOC_C) -#error "MBEDTLS_MEMORY_DEBUG defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PEM_PARSE_C) && !defined(MBEDTLS_BASE64_C) -#error "MBEDTLS_PEM_PARSE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PEM_WRITE_C) && !defined(MBEDTLS_BASE64_C) -#error "MBEDTLS_PEM_WRITE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PK_C) && \ - !defined(MBEDTLS_RSA_C) && !defined(MBEDTLS_PK_HAVE_ECC_KEYS) -#error "MBEDTLS_PK_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PK_PARSE_C) && \ - (!defined(MBEDTLS_ASN1_PARSE_C) || \ - !defined(MBEDTLS_OID_C) || \ - !defined(MBEDTLS_PK_C)) -#error "MBEDTLS_PK_PARSE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PK_WRITE_C) && \ - (!defined(MBEDTLS_ASN1_WRITE_C) || \ - !defined(MBEDTLS_OID_C) || \ - !defined(MBEDTLS_PK_C)) -#error "MBEDTLS_PK_WRITE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_EXIT_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_EXIT_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_EXIT_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_EXIT_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_EXIT_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_EXIT) ||\ - defined(MBEDTLS_PLATFORM_EXIT_ALT) ) -#error "MBEDTLS_PLATFORM_EXIT_MACRO and MBEDTLS_PLATFORM_STD_EXIT/MBEDTLS_PLATFORM_EXIT_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_SETBUF_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_SETBUF_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_SETBUF_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_SETBUF_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_SETBUF_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_SETBUF) ||\ - defined(MBEDTLS_PLATFORM_SETBUF_ALT) ) -#error "MBEDTLS_PLATFORM_SETBUF_MACRO and MBEDTLS_PLATFORM_STD_SETBUF/MBEDTLS_PLATFORM_SETBUF_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_TIME_ALT) &&\ - ( !defined(MBEDTLS_PLATFORM_C) ||\ - !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_TIME_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_TIME_MACRO) &&\ - ( !defined(MBEDTLS_PLATFORM_C) ||\ - !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_TIME_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO) &&\ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_MS_TIME_ALT) && \ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_MS_TIME_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_TIME_TYPE_MACRO) &&\ - ( !defined(MBEDTLS_PLATFORM_C) ||\ - !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_TIME_TYPE_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_TIME_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_TIME) ||\ - defined(MBEDTLS_PLATFORM_TIME_ALT) ) -#error "MBEDTLS_PLATFORM_TIME_MACRO and MBEDTLS_PLATFORM_STD_TIME/MBEDTLS_PLATFORM_TIME_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_TIME_TYPE_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_TIME) ||\ - defined(MBEDTLS_PLATFORM_TIME_ALT) ) -#error "MBEDTLS_PLATFORM_TIME_TYPE_MACRO and MBEDTLS_PLATFORM_STD_TIME/MBEDTLS_PLATFORM_TIME_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_FPRINTF_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_FPRINTF_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_FPRINTF_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_FPRINTF_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_FPRINTF_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_FPRINTF) ||\ - defined(MBEDTLS_PLATFORM_FPRINTF_ALT) ) -#error "MBEDTLS_PLATFORM_FPRINTF_MACRO and MBEDTLS_PLATFORM_STD_FPRINTF/MBEDTLS_PLATFORM_FPRINTF_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_FREE_MACRO) &&\ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_PLATFORM_MEMORY) ) -#error "MBEDTLS_PLATFORM_FREE_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_FREE_MACRO) &&\ - defined(MBEDTLS_PLATFORM_STD_FREE) -#error "MBEDTLS_PLATFORM_FREE_MACRO and MBEDTLS_PLATFORM_STD_FREE cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_FREE_MACRO) && !defined(MBEDTLS_PLATFORM_CALLOC_MACRO) -#error "MBEDTLS_PLATFORM_CALLOC_MACRO must be defined if MBEDTLS_PLATFORM_FREE_MACRO is" -#endif - -#if defined(MBEDTLS_PLATFORM_CALLOC_MACRO) &&\ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_PLATFORM_MEMORY) ) -#error "MBEDTLS_PLATFORM_CALLOC_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_CALLOC_MACRO) &&\ - defined(MBEDTLS_PLATFORM_STD_CALLOC) -#error "MBEDTLS_PLATFORM_CALLOC_MACRO and MBEDTLS_PLATFORM_STD_CALLOC cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_CALLOC_MACRO) && !defined(MBEDTLS_PLATFORM_FREE_MACRO) -#error "MBEDTLS_PLATFORM_FREE_MACRO must be defined if MBEDTLS_PLATFORM_CALLOC_MACRO is" -#endif - -#if defined(MBEDTLS_PLATFORM_MEMORY) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_MEMORY defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_PRINTF_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_PRINTF_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_PRINTF_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_PRINTF_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_PRINTF_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_PRINTF) ||\ - defined(MBEDTLS_PLATFORM_PRINTF_ALT) ) -#error "MBEDTLS_PLATFORM_PRINTF_MACRO and MBEDTLS_PLATFORM_STD_PRINTF/MBEDTLS_PLATFORM_PRINTF_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_SNPRINTF_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_SNPRINTF_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_SNPRINTF_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_SNPRINTF_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_SNPRINTF) ||\ - defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) ) -#error "MBEDTLS_PLATFORM_SNPRINTF_MACRO and MBEDTLS_PLATFORM_STD_SNPRINTF/MBEDTLS_PLATFORM_SNPRINTF_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_VSNPRINTF_ALT) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_VSNPRINTF_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_VSNPRINTF_MACRO) && !defined(MBEDTLS_PLATFORM_C) -#error "MBEDTLS_PLATFORM_VSNPRINTF_MACRO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_VSNPRINTF_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_VSNPRINTF) ||\ - defined(MBEDTLS_PLATFORM_VSNPRINTF_ALT) ) -#error "MBEDTLS_PLATFORM_VSNPRINTF_MACRO and MBEDTLS_PLATFORM_STD_VSNPRINTF/MBEDTLS_PLATFORM_VSNPRINTF_ALT cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_MEM_HDR) &&\ - !defined(MBEDTLS_PLATFORM_NO_STD_FUNCTIONS) -#error "MBEDTLS_PLATFORM_STD_MEM_HDR defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_CALLOC) && !defined(MBEDTLS_PLATFORM_MEMORY) -#error "MBEDTLS_PLATFORM_STD_CALLOC defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_FREE) && !defined(MBEDTLS_PLATFORM_MEMORY) -#error "MBEDTLS_PLATFORM_STD_FREE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_EXIT) &&\ - !defined(MBEDTLS_PLATFORM_EXIT_ALT) -#error "MBEDTLS_PLATFORM_STD_EXIT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_TIME) &&\ - ( !defined(MBEDTLS_PLATFORM_TIME_ALT) ||\ - !defined(MBEDTLS_HAVE_TIME) ) -#error "MBEDTLS_PLATFORM_STD_TIME defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_FPRINTF) &&\ - !defined(MBEDTLS_PLATFORM_FPRINTF_ALT) -#error "MBEDTLS_PLATFORM_STD_FPRINTF defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_PRINTF) &&\ - !defined(MBEDTLS_PLATFORM_PRINTF_ALT) -#error "MBEDTLS_PLATFORM_STD_PRINTF defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_SNPRINTF) &&\ - !defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) -#error "MBEDTLS_PLATFORM_STD_SNPRINTF defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_ENTROPY_NV_SEED) &&\ - ( !defined(MBEDTLS_PLATFORM_C) || !defined(MBEDTLS_ENTROPY_C) ) -#error "MBEDTLS_ENTROPY_NV_SEED defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_NV_SEED_ALT) &&\ - !defined(MBEDTLS_ENTROPY_NV_SEED) -#error "MBEDTLS_PLATFORM_NV_SEED_ALT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_NV_SEED_READ) &&\ - !defined(MBEDTLS_PLATFORM_NV_SEED_ALT) -#error "MBEDTLS_PLATFORM_STD_NV_SEED_READ defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_STD_NV_SEED_WRITE) &&\ - !defined(MBEDTLS_PLATFORM_NV_SEED_ALT) -#error "MBEDTLS_PLATFORM_STD_NV_SEED_WRITE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PLATFORM_NV_SEED_READ_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_NV_SEED_READ) ||\ - defined(MBEDTLS_PLATFORM_NV_SEED_ALT) ) -#error "MBEDTLS_PLATFORM_NV_SEED_READ_MACRO and MBEDTLS_PLATFORM_STD_NV_SEED_READ cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO) &&\ - ( defined(MBEDTLS_PLATFORM_STD_NV_SEED_WRITE) ||\ - defined(MBEDTLS_PLATFORM_NV_SEED_ALT) ) -#error "MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO and MBEDTLS_PLATFORM_STD_NV_SEED_WRITE cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_C) && \ - !( ( ( defined(MBEDTLS_CTR_DRBG_C) || defined(MBEDTLS_HMAC_DRBG_C) ) && \ - defined(MBEDTLS_ENTROPY_C) ) || \ - defined(MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG) ) -#error "MBEDTLS_PSA_CRYPTO_C defined, but not all prerequisites (missing RNG)" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_C) && defined(PSA_HAVE_SOFT_BLOCK_MODE) && \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) && !defined(MBEDTLS_CIPHER_C) -#error "MBEDTLS_PSA_CRYPTO_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_SPM) && !defined(MBEDTLS_PSA_CRYPTO_C) -#error "MBEDTLS_PSA_CRYPTO_SPM defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) && \ - ! ( defined(MBEDTLS_PSA_CRYPTO_C) && \ - defined(MBEDTLS_PSA_CRYPTO_STORAGE_C) ) -#error "MBEDTLS_PSA_CRYPTO_SE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) -#if defined(MBEDTLS_DEPRECATED_REMOVED) -#error "MBEDTLS_PSA_CRYPTO_SE_C is deprecated and will be removed in a future version of Mbed TLS" -#elif defined(MBEDTLS_DEPRECATED_WARNING) -#warning "MBEDTLS_PSA_CRYPTO_SE_C is deprecated and will be removed in a future version of Mbed TLS" -#endif -#endif /* MBEDTLS_PSA_CRYPTO_SE_C */ - -#if defined(MBEDTLS_PSA_CRYPTO_STORAGE_C) && \ - ! defined(MBEDTLS_PSA_CRYPTO_C) -#error "MBEDTLS_PSA_CRYPTO_STORAGE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PSA_INJECT_ENTROPY) && \ - !( defined(MBEDTLS_PSA_CRYPTO_STORAGE_C) && \ - defined(MBEDTLS_ENTROPY_NV_SEED) ) -#error "MBEDTLS_PSA_INJECT_ENTROPY defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_PSA_INJECT_ENTROPY) && \ - !defined(MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES) -#error "MBEDTLS_PSA_INJECT_ENTROPY is not compatible with actual entropy sources" -#endif - -#if defined(MBEDTLS_PSA_INJECT_ENTROPY) && \ - defined(MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG) -#error "MBEDTLS_PSA_INJECT_ENTROPY is not compatible with MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG" -#endif - -#if defined(MBEDTLS_PSA_KEY_STORE_DYNAMIC) && \ - defined(MBEDTLS_PSA_STATIC_KEY_SLOTS) -#error "MBEDTLS_PSA_KEY_STORE_DYNAMIC and MBEDTLS_PSA_STATIC_KEY_SLOTS cannot be defined simultaneously" -#endif - -#if defined(MBEDTLS_PSA_ITS_FILE_C) && \ - !defined(MBEDTLS_FS_IO) -#error "MBEDTLS_PSA_ITS_FILE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_RSA_C) && ( !defined(MBEDTLS_BIGNUM_C) || \ - !defined(MBEDTLS_OID_C) ) -#error "MBEDTLS_RSA_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_RSA_C) && ( !defined(MBEDTLS_PKCS1_V21) && \ - !defined(MBEDTLS_PKCS1_V15) ) -#error "MBEDTLS_RSA_C defined, but none of the PKCS1 versions enabled" -#endif - -#if defined(MBEDTLS_X509_RSASSA_PSS_SUPPORT) && \ - ( !defined(MBEDTLS_RSA_C) || !defined(MBEDTLS_PKCS1_V21) ) -#error "MBEDTLS_X509_RSASSA_PSS_SUPPORT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT) && \ - defined(MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY) -#error "Must only define one of MBEDTLS_SHA512_USE_A64_CRYPTO_*" -#endif - -#if defined(MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT) || \ - defined(MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY) -#if !defined(MBEDTLS_SHA512_C) -#error "MBEDTLS_SHA512_USE_A64_CRYPTO_* defined without MBEDTLS_SHA512_C" -#endif -#if defined(MBEDTLS_SHA512_ALT) || defined(MBEDTLS_SHA512_PROCESS_ALT) -#error "MBEDTLS_SHA512_*ALT can't be used with MBEDTLS_SHA512_USE_A64_CRYPTO_*" -#endif - -#endif /* MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT || MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY */ - -#if defined(MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY) && !defined(__aarch64__) -#error "MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY defined on non-Aarch64 system" -#endif - -#if defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT) && \ - defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY) -#error "Must only define one of MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_*" -#endif - -#if defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT) || \ - defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY) -#if !defined(MBEDTLS_SHA256_C) -#error "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_* defined without MBEDTLS_SHA256_C" -#endif -#if defined(MBEDTLS_SHA256_ALT) || defined(MBEDTLS_SHA256_PROCESS_ALT) -#error "MBEDTLS_SHA256_*ALT can't be used with MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_*" -#endif - -#endif - -#if defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY) && !defined(MBEDTLS_ARCH_IS_ARMV8_A) -#error "MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY defined on non-Armv8-A system" -#endif - -/* TLS 1.3 requires separate HKDF parts from PSA, - * and at least one ciphersuite, so at least SHA-256 or SHA-384 - * from PSA to use with HKDF. - * - * Note: for dependencies common with TLS 1.2 (running handshake hash), - * see MBEDTLS_SSL_TLS_C. */ -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && \ - !(defined(MBEDTLS_PSA_CRYPTO_CLIENT) && \ - defined(PSA_WANT_ALG_HKDF_EXTRACT) && \ - defined(PSA_WANT_ALG_HKDF_EXPAND) && \ - (defined(PSA_WANT_ALG_SHA_256) || defined(PSA_WANT_ALG_SHA_384))) -#error "MBEDTLS_SSL_PROTO_TLS1_3 defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED) -#if !( (defined(PSA_WANT_ALG_ECDH) || defined(PSA_WANT_ALG_FFDH)) && \ - defined(MBEDTLS_X509_CRT_PARSE_C) && \ - ( defined(MBEDTLS_PK_CAN_ECDSA_SIGN) || defined(MBEDTLS_PKCS1_V21) ) ) -#error "MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED defined, but not all prerequisites" -#endif -#endif - -#if defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED) -#if !( defined(PSA_WANT_ALG_ECDH) || defined(PSA_WANT_ALG_FFDH) ) -#error "MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED defined, but not all prerequisites" -#endif -#endif - -/* - * The current implementation of TLS 1.3 requires MBEDTLS_SSL_KEEP_PEER_CERTIFICATE. - */ -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && !defined(MBEDTLS_SSL_KEEP_PEER_CERTIFICATE) -#error "MBEDTLS_SSL_PROTO_TLS1_3 defined without MBEDTLS_SSL_KEEP_PEER_CERTIFICATE" -#endif - -#if defined(MBEDTLS_SSL_PROTO_TLS1_2) && \ - !(defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) ) -#error "One or more versions of the TLS protocol are enabled " \ - "but no key exchange methods defined with MBEDTLS_KEY_EXCHANGE_xxxx" -#endif - -#if defined(MBEDTLS_SSL_EARLY_DATA) && \ - ( !defined(MBEDTLS_SSL_SESSION_TICKETS) || \ - ( !defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED) && \ - !defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED) ) ) -#error "MBEDTLS_SSL_EARLY_DATA defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_EARLY_DATA) && defined(MBEDTLS_SSL_SRV_C) && \ - defined(MBEDTLS_SSL_MAX_EARLY_DATA_SIZE) && \ - ((MBEDTLS_SSL_MAX_EARLY_DATA_SIZE < 0) || \ - (MBEDTLS_SSL_MAX_EARLY_DATA_SIZE > UINT32_MAX)) -#error "MBEDTLS_SSL_MAX_EARLY_DATA_SIZE must be in the range(0..UINT32_MAX)" -#endif - -#if defined(MBEDTLS_SSL_PROTO_DTLS) && \ - !defined(MBEDTLS_SSL_PROTO_TLS1_2) -#error "MBEDTLS_SSL_PROTO_DTLS defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_CLI_C) && !defined(MBEDTLS_SSL_TLS_C) -#error "MBEDTLS_SSL_CLI_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_ASYNC_PRIVATE) && !defined(MBEDTLS_X509_CRT_PARSE_C) -#error "MBEDTLS_SSL_ASYNC_PRIVATE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TLS_C) && !(defined(MBEDTLS_CIPHER_C) || \ - defined(MBEDTLS_USE_PSA_CRYPTO)) -#error "MBEDTLS_SSL_TLS_C defined, but not all prerequisites" -#endif - -/* TLS 1.2 and 1.3 require SHA-256 or SHA-384 (running handshake hash) */ -#if defined(MBEDTLS_SSL_TLS_C) -#if defined(MBEDTLS_USE_PSA_CRYPTO) -#if !(defined(PSA_WANT_ALG_SHA_256) || defined(PSA_WANT_ALG_SHA_384)) -#error "MBEDTLS_SSL_TLS_C defined, but not all prerequisites" -#endif -#else /* MBEDTLS_USE_PSA_CRYPTO */ -#if !defined(MBEDTLS_MD_C) || \ - !(defined(MBEDTLS_MD_CAN_SHA256) || defined(MBEDTLS_MD_CAN_SHA384)) -#error "MBEDTLS_SSL_TLS_C defined, but not all prerequisites" -#endif -#endif /* MBEDTLS_USE_PSA_CRYPTO */ -#endif /* MBEDTLS_SSL_TLS_C */ - -#if defined(MBEDTLS_SSL_SRV_C) && !defined(MBEDTLS_SSL_TLS_C) -#error "MBEDTLS_SSL_SRV_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TLS_C) && \ - !( defined(MBEDTLS_SSL_PROTO_TLS1_2) || defined(MBEDTLS_SSL_PROTO_TLS1_3) ) -#error "MBEDTLS_SSL_TLS_C defined, but no protocols are active" -#endif - -#if defined(MBEDTLS_SSL_DTLS_HELLO_VERIFY) && !defined(MBEDTLS_SSL_PROTO_DTLS) -#error "MBEDTLS_SSL_DTLS_HELLO_VERIFY defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE) && \ - !defined(MBEDTLS_SSL_DTLS_HELLO_VERIFY) -#error "MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY) && \ - ( !defined(MBEDTLS_SSL_TLS_C) || !defined(MBEDTLS_SSL_PROTO_DTLS) ) -#error "MBEDTLS_SSL_DTLS_ANTI_REPLAY defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) && \ - ( !defined(MBEDTLS_SSL_TLS_C) || !defined(MBEDTLS_SSL_PROTO_DTLS) ) -#error "MBEDTLS_SSL_DTLS_CONNECTION_ID defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) && \ - defined(MBEDTLS_SSL_CID_IN_LEN_MAX) && \ - MBEDTLS_SSL_CID_IN_LEN_MAX > 255 -#error "MBEDTLS_SSL_CID_IN_LEN_MAX too large (max 255)" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) && \ - defined(MBEDTLS_SSL_CID_OUT_LEN_MAX) && \ - MBEDTLS_SSL_CID_OUT_LEN_MAX > 255 -#error "MBEDTLS_SSL_CID_OUT_LEN_MAX too large (max 255)" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT) && \ - !defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) -#error "MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT) && MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT != 0 -#if defined(MBEDTLS_DEPRECATED_REMOVED) -#error "MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT is deprecated and will be removed in a future version of Mbed TLS" -#elif defined(MBEDTLS_DEPRECATED_WARNING) -#warning "MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT is deprecated and will be removed in a future version of Mbed TLS" -#endif -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT && MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT != 0 */ - -#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC) && \ - !defined(MBEDTLS_SSL_PROTO_TLS1_2) -#error "MBEDTLS_SSL_ENCRYPT_THEN_MAC defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET) && \ - !defined(MBEDTLS_SSL_PROTO_TLS1_2) -#error "MBEDTLS_SSL_EXTENDED_MASTER_SECRET defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_RENEGOTIATION) && \ - !defined(MBEDTLS_SSL_PROTO_TLS1_2) -#error "MBEDTLS_SSL_RENEGOTIATION defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TICKET_C) && ( !defined(MBEDTLS_CIPHER_C) && \ - !defined(MBEDTLS_USE_PSA_CRYPTO) ) -#error "MBEDTLS_SSL_TICKET_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TICKET_C) && \ - !( defined(MBEDTLS_SSL_HAVE_CCM) || defined(MBEDTLS_SSL_HAVE_GCM) || \ - defined(MBEDTLS_SSL_HAVE_CHACHAPOLY) ) -#error "MBEDTLS_SSL_TICKET_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH) && \ - MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH >= 256 -#error "MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH must be less than 256" -#endif - -#if defined(MBEDTLS_SSL_SERVER_NAME_INDICATION) && \ - !defined(MBEDTLS_X509_CRT_PARSE_C) -#error "MBEDTLS_SSL_SERVER_NAME_INDICATION defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_THREADING_PTHREAD) -#if !defined(MBEDTLS_THREADING_C) || defined(MBEDTLS_THREADING_IMPL) -#error "MBEDTLS_THREADING_PTHREAD defined, but not all prerequisites" -#endif -#define MBEDTLS_THREADING_IMPL // undef at the end of this paragraph -#endif -#if defined(MBEDTLS_THREADING_ALT) -#if !defined(MBEDTLS_THREADING_C) || defined(MBEDTLS_THREADING_IMPL) -#error "MBEDTLS_THREADING_ALT defined, but not all prerequisites" -#endif -#define MBEDTLS_THREADING_IMPL // undef at the end of this paragraph -#endif -#if defined(MBEDTLS_THREADING_C) && !defined(MBEDTLS_THREADING_IMPL) -#error "MBEDTLS_THREADING_C defined, single threading implementation required" -#endif -#undef MBEDTLS_THREADING_IMPL // temporary macro defined above - -#if defined(MBEDTLS_USE_PSA_CRYPTO) && !defined(MBEDTLS_PSA_CRYPTO_CLIENT) -#error "MBEDTLS_USE_PSA_CRYPTO defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_VERSION_FEATURES) && !defined(MBEDTLS_VERSION_C) -#error "MBEDTLS_VERSION_FEATURES defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_USE_C) && \ - (!defined(MBEDTLS_OID_C) || !defined(MBEDTLS_ASN1_PARSE_C) || \ - !defined(MBEDTLS_PK_PARSE_C) || \ - ( !defined(MBEDTLS_MD_C) && !defined(MBEDTLS_USE_PSA_CRYPTO) ) ) -#error "MBEDTLS_X509_USE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CREATE_C) && \ - (!defined(MBEDTLS_OID_C) || !defined(MBEDTLS_ASN1_WRITE_C) || \ - !defined(MBEDTLS_PK_PARSE_C) || \ - ( !defined(MBEDTLS_MD_C) && !defined(MBEDTLS_USE_PSA_CRYPTO) ) ) -#error "MBEDTLS_X509_CREATE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CRT_PARSE_C) && ( !defined(MBEDTLS_X509_USE_C) ) -#error "MBEDTLS_X509_CRT_PARSE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CRL_PARSE_C) && ( !defined(MBEDTLS_X509_USE_C) ) -#error "MBEDTLS_X509_CRL_PARSE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CSR_PARSE_C) && ( !defined(MBEDTLS_X509_USE_C) ) -#error "MBEDTLS_X509_CSR_PARSE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CRT_WRITE_C) && ( !defined(MBEDTLS_X509_CREATE_C) ) -#error "MBEDTLS_X509_CRT_WRITE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_CSR_WRITE_C) && ( !defined(MBEDTLS_X509_CREATE_C) ) -#error "MBEDTLS_X509_CSR_WRITE_C defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK) && \ - ( !defined(MBEDTLS_X509_CRT_PARSE_C) ) -#error "MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_HAVE_INT32) && defined(MBEDTLS_HAVE_INT64) -#error "MBEDTLS_HAVE_INT32 and MBEDTLS_HAVE_INT64 cannot be defined simultaneously" -#endif /* MBEDTLS_HAVE_INT32 && MBEDTLS_HAVE_INT64 */ - -#if ( defined(MBEDTLS_HAVE_INT32) || defined(MBEDTLS_HAVE_INT64) ) && \ - defined(MBEDTLS_HAVE_ASM) -#error "MBEDTLS_HAVE_INT32/MBEDTLS_HAVE_INT64 and MBEDTLS_HAVE_ASM cannot be defined simultaneously" -#endif /* (MBEDTLS_HAVE_INT32 || MBEDTLS_HAVE_INT64) && MBEDTLS_HAVE_ASM */ - -#if defined(MBEDTLS_SSL_DTLS_SRTP) && ( !defined(MBEDTLS_SSL_PROTO_DTLS) ) -#error "MBEDTLS_SSL_DTLS_SRTP defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH) && ( !defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH) ) -#error "MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_RECORD_SIZE_LIMIT) && ( !defined(MBEDTLS_SSL_PROTO_TLS1_3) ) -#error "MBEDTLS_SSL_RECORD_SIZE_LIMIT defined, but not all prerequisites" -#endif - -#if defined(MBEDTLS_SSL_CONTEXT_SERIALIZATION) && \ - !( defined(MBEDTLS_SSL_HAVE_CCM) || defined(MBEDTLS_SSL_HAVE_GCM) || \ - defined(MBEDTLS_SSL_HAVE_CHACHAPOLY) ) -#error "MBEDTLS_SSL_CONTEXT_SERIALIZATION defined, but not all prerequisites" -#endif - -/* Reject attempts to enable options that have been removed and that could - * cause a build to succeed but with features removed. */ - -#if defined(MBEDTLS_HAVEGE_C) //no-check-names -#error "MBEDTLS_HAVEGE_C was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/2599" -#endif - -#if defined(MBEDTLS_SSL_HW_RECORD_ACCEL) //no-check-names -#error "MBEDTLS_SSL_HW_RECORD_ACCEL was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4031" -#endif - -#if defined(MBEDTLS_SSL_PROTO_SSL3) //no-check-names -#error "MBEDTLS_SSL_PROTO_SSL3 (SSL v3.0 support) was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4031" -#endif - -#if defined(MBEDTLS_SSL_SRV_SUPPORT_SSLV2_CLIENT_HELLO) //no-check-names -#error "MBEDTLS_SSL_SRV_SUPPORT_SSLV2_CLIENT_HELLO (SSL v2 ClientHello support) was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4031" -#endif - -#if defined(MBEDTLS_SSL_TRUNCATED_HMAC_COMPAT) //no-check-names -#error "MBEDTLS_SSL_TRUNCATED_HMAC_COMPAT (compatibility with the buggy implementation of truncated HMAC in Mbed TLS up to 2.7) was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4031" -#endif - -#if defined(MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES) //no-check-names -#error "MBEDTLS_TLS_DEFAULT_ALLOW_SHA1_IN_CERTIFICATES was removed in Mbed TLS 3.0. See the ChangeLog entry if you really need SHA-1-signed certificates." -#endif - -#if defined(MBEDTLS_ZLIB_SUPPORT) //no-check-names -#error "MBEDTLS_ZLIB_SUPPORT was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4031" -#endif - -#if defined(MBEDTLS_CHECK_PARAMS) //no-check-names -#error "MBEDTLS_CHECK_PARAMS was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4313" -#endif - -#if defined(MBEDTLS_SSL_CID_PADDING_GRANULARITY) //no-check-names -#error "MBEDTLS_SSL_CID_PADDING_GRANULARITY was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4335" -#endif - -#if defined(MBEDTLS_SSL_TLS1_3_PADDING_GRANULARITY) //no-check-names -#error "MBEDTLS_SSL_TLS1_3_PADDING_GRANULARITY was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4335" -#endif - -#if defined(MBEDTLS_SSL_TRUNCATED_HMAC) //no-check-names -#error "MBEDTLS_SSL_TRUNCATED_HMAC was removed in Mbed TLS 3.0. See https://github.com/Mbed-TLS/mbedtls/issues/4341" -#endif - -#if defined(MBEDTLS_PKCS7_C) && ( ( !defined(MBEDTLS_ASN1_PARSE_C) ) || \ - ( !defined(MBEDTLS_OID_C) ) || ( !defined(MBEDTLS_PK_PARSE_C) ) || \ - ( !defined(MBEDTLS_X509_CRT_PARSE_C) ) || \ - ( !defined(MBEDTLS_X509_CRL_PARSE_C) ) || \ - ( !defined(MBEDTLS_MD_C) ) ) -#error "MBEDTLS_PKCS7_C is defined, but not all prerequisites" -#endif - -/* - * Avoid warning from -pedantic. This is a convenient place for this - * workaround since this is included by every single file before the - * #if defined(MBEDTLS_xxx_C) that results in empty translation units. - */ -typedef int mbedtls_iso_c_forbids_empty_translation_units; - -/* *INDENT-ON* */ -#endif /* MBEDTLS_CHECK_CONFIG_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/cipher.h b/android/mbedtls/arm64-v8a/include/mbedtls/cipher.h deleted file mode 100644 index e54715215..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/cipher.h +++ /dev/null @@ -1,1250 +0,0 @@ -/** - * \file cipher.h - * - * \brief This file contains an abstraction interface for use with the cipher - * primitives provided by the library. It provides a common interface to all of - * the available cipher operations. - * - * \author Adriaan de Jong - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CIPHER_H -#define MBEDTLS_CIPHER_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include "mbedtls/platform_util.h" - -#if defined(MBEDTLS_GCM_C) || defined(MBEDTLS_CCM_C) || defined(MBEDTLS_CHACHAPOLY_C) -#define MBEDTLS_CIPHER_MODE_AEAD -#endif - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -#define MBEDTLS_CIPHER_MODE_WITH_PADDING -#endif - -#if defined(MBEDTLS_CIPHER_NULL_CIPHER) || \ - defined(MBEDTLS_CHACHA20_C) -#define MBEDTLS_CIPHER_MODE_STREAM -#endif - -/** The selected feature is not available. */ -#define MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE -0x6080 -/** Bad input parameters. */ -#define MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA -0x6100 -/** Failed to allocate memory. */ -#define MBEDTLS_ERR_CIPHER_ALLOC_FAILED -0x6180 -/** Input data contains invalid padding and is rejected. */ -#define MBEDTLS_ERR_CIPHER_INVALID_PADDING -0x6200 -/** Decryption of block requires a full block. */ -#define MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED -0x6280 -/** Authentication failed (for AEAD modes). */ -#define MBEDTLS_ERR_CIPHER_AUTH_FAILED -0x6300 -/** The context is invalid. For example, because it was freed. */ -#define MBEDTLS_ERR_CIPHER_INVALID_CONTEXT -0x6380 - -#define MBEDTLS_CIPHER_VARIABLE_IV_LEN 0x01 /**< Cipher accepts IVs of variable length. */ -#define MBEDTLS_CIPHER_VARIABLE_KEY_LEN 0x02 /**< Cipher accepts keys of variable length. */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Supported cipher types. - * - * \warning DES/3DES are considered weak ciphers and their use - * constitutes a security risk. We recommend considering stronger - * ciphers instead. - */ -typedef enum { - MBEDTLS_CIPHER_ID_NONE = 0, /**< Placeholder to mark the end of cipher ID lists. */ - MBEDTLS_CIPHER_ID_NULL, /**< The identity cipher, treated as a stream cipher. */ - MBEDTLS_CIPHER_ID_AES, /**< The AES cipher. */ - MBEDTLS_CIPHER_ID_DES, /**< The DES cipher. \warning DES is considered weak. */ - MBEDTLS_CIPHER_ID_3DES, /**< The Triple DES cipher. \warning 3DES is considered weak. */ - MBEDTLS_CIPHER_ID_CAMELLIA, /**< The Camellia cipher. */ - MBEDTLS_CIPHER_ID_ARIA, /**< The Aria cipher. */ - MBEDTLS_CIPHER_ID_CHACHA20, /**< The ChaCha20 cipher. */ -} mbedtls_cipher_id_t; - -/** - * \brief Supported {cipher type, cipher mode} pairs. - * - * \warning DES/3DES are considered weak ciphers and their use - * constitutes a security risk. We recommend considering stronger - * ciphers instead. - */ -typedef enum { - MBEDTLS_CIPHER_NONE = 0, /**< Placeholder to mark the end of cipher-pair lists. */ - MBEDTLS_CIPHER_NULL, /**< The identity stream cipher. */ - MBEDTLS_CIPHER_AES_128_ECB, /**< AES cipher with 128-bit ECB mode. */ - MBEDTLS_CIPHER_AES_192_ECB, /**< AES cipher with 192-bit ECB mode. */ - MBEDTLS_CIPHER_AES_256_ECB, /**< AES cipher with 256-bit ECB mode. */ - MBEDTLS_CIPHER_AES_128_CBC, /**< AES cipher with 128-bit CBC mode. */ - MBEDTLS_CIPHER_AES_192_CBC, /**< AES cipher with 192-bit CBC mode. */ - MBEDTLS_CIPHER_AES_256_CBC, /**< AES cipher with 256-bit CBC mode. */ - MBEDTLS_CIPHER_AES_128_CFB128, /**< AES cipher with 128-bit CFB128 mode. */ - MBEDTLS_CIPHER_AES_192_CFB128, /**< AES cipher with 192-bit CFB128 mode. */ - MBEDTLS_CIPHER_AES_256_CFB128, /**< AES cipher with 256-bit CFB128 mode. */ - MBEDTLS_CIPHER_AES_128_CTR, /**< AES cipher with 128-bit CTR mode. */ - MBEDTLS_CIPHER_AES_192_CTR, /**< AES cipher with 192-bit CTR mode. */ - MBEDTLS_CIPHER_AES_256_CTR, /**< AES cipher with 256-bit CTR mode. */ - MBEDTLS_CIPHER_AES_128_GCM, /**< AES cipher with 128-bit GCM mode. */ - MBEDTLS_CIPHER_AES_192_GCM, /**< AES cipher with 192-bit GCM mode. */ - MBEDTLS_CIPHER_AES_256_GCM, /**< AES cipher with 256-bit GCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_ECB, /**< Camellia cipher with 128-bit ECB mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_ECB, /**< Camellia cipher with 192-bit ECB mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_ECB, /**< Camellia cipher with 256-bit ECB mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_CBC, /**< Camellia cipher with 128-bit CBC mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_CBC, /**< Camellia cipher with 192-bit CBC mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_CBC, /**< Camellia cipher with 256-bit CBC mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_CFB128, /**< Camellia cipher with 128-bit CFB128 mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_CFB128, /**< Camellia cipher with 192-bit CFB128 mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_CFB128, /**< Camellia cipher with 256-bit CFB128 mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_CTR, /**< Camellia cipher with 128-bit CTR mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_CTR, /**< Camellia cipher with 192-bit CTR mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_CTR, /**< Camellia cipher with 256-bit CTR mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_GCM, /**< Camellia cipher with 128-bit GCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_GCM, /**< Camellia cipher with 192-bit GCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_GCM, /**< Camellia cipher with 256-bit GCM mode. */ - MBEDTLS_CIPHER_DES_ECB, /**< DES cipher with ECB mode. \warning DES is considered weak. */ - MBEDTLS_CIPHER_DES_CBC, /**< DES cipher with CBC mode. \warning DES is considered weak. */ - MBEDTLS_CIPHER_DES_EDE_ECB, /**< DES cipher with EDE ECB mode. \warning 3DES is considered weak. */ - MBEDTLS_CIPHER_DES_EDE_CBC, /**< DES cipher with EDE CBC mode. \warning 3DES is considered weak. */ - MBEDTLS_CIPHER_DES_EDE3_ECB, /**< DES cipher with EDE3 ECB mode. \warning 3DES is considered weak. */ - MBEDTLS_CIPHER_DES_EDE3_CBC, /**< DES cipher with EDE3 CBC mode. \warning 3DES is considered weak. */ - MBEDTLS_CIPHER_AES_128_CCM, /**< AES cipher with 128-bit CCM mode. */ - MBEDTLS_CIPHER_AES_192_CCM, /**< AES cipher with 192-bit CCM mode. */ - MBEDTLS_CIPHER_AES_256_CCM, /**< AES cipher with 256-bit CCM mode. */ - MBEDTLS_CIPHER_AES_128_CCM_STAR_NO_TAG, /**< AES cipher with 128-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_AES_192_CCM_STAR_NO_TAG, /**< AES cipher with 192-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_AES_256_CCM_STAR_NO_TAG, /**< AES cipher with 256-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_CCM, /**< Camellia cipher with 128-bit CCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_CCM, /**< Camellia cipher with 192-bit CCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_CCM, /**< Camellia cipher with 256-bit CCM mode. */ - MBEDTLS_CIPHER_CAMELLIA_128_CCM_STAR_NO_TAG, /**< Camellia cipher with 128-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_CAMELLIA_192_CCM_STAR_NO_TAG, /**< Camellia cipher with 192-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_CAMELLIA_256_CCM_STAR_NO_TAG, /**< Camellia cipher with 256-bit CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_ARIA_128_ECB, /**< Aria cipher with 128-bit key and ECB mode. */ - MBEDTLS_CIPHER_ARIA_192_ECB, /**< Aria cipher with 192-bit key and ECB mode. */ - MBEDTLS_CIPHER_ARIA_256_ECB, /**< Aria cipher with 256-bit key and ECB mode. */ - MBEDTLS_CIPHER_ARIA_128_CBC, /**< Aria cipher with 128-bit key and CBC mode. */ - MBEDTLS_CIPHER_ARIA_192_CBC, /**< Aria cipher with 192-bit key and CBC mode. */ - MBEDTLS_CIPHER_ARIA_256_CBC, /**< Aria cipher with 256-bit key and CBC mode. */ - MBEDTLS_CIPHER_ARIA_128_CFB128, /**< Aria cipher with 128-bit key and CFB-128 mode. */ - MBEDTLS_CIPHER_ARIA_192_CFB128, /**< Aria cipher with 192-bit key and CFB-128 mode. */ - MBEDTLS_CIPHER_ARIA_256_CFB128, /**< Aria cipher with 256-bit key and CFB-128 mode. */ - MBEDTLS_CIPHER_ARIA_128_CTR, /**< Aria cipher with 128-bit key and CTR mode. */ - MBEDTLS_CIPHER_ARIA_192_CTR, /**< Aria cipher with 192-bit key and CTR mode. */ - MBEDTLS_CIPHER_ARIA_256_CTR, /**< Aria cipher with 256-bit key and CTR mode. */ - MBEDTLS_CIPHER_ARIA_128_GCM, /**< Aria cipher with 128-bit key and GCM mode. */ - MBEDTLS_CIPHER_ARIA_192_GCM, /**< Aria cipher with 192-bit key and GCM mode. */ - MBEDTLS_CIPHER_ARIA_256_GCM, /**< Aria cipher with 256-bit key and GCM mode. */ - MBEDTLS_CIPHER_ARIA_128_CCM, /**< Aria cipher with 128-bit key and CCM mode. */ - MBEDTLS_CIPHER_ARIA_192_CCM, /**< Aria cipher with 192-bit key and CCM mode. */ - MBEDTLS_CIPHER_ARIA_256_CCM, /**< Aria cipher with 256-bit key and CCM mode. */ - MBEDTLS_CIPHER_ARIA_128_CCM_STAR_NO_TAG, /**< Aria cipher with 128-bit key and CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_ARIA_192_CCM_STAR_NO_TAG, /**< Aria cipher with 192-bit key and CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_ARIA_256_CCM_STAR_NO_TAG, /**< Aria cipher with 256-bit key and CCM_STAR_NO_TAG mode. */ - MBEDTLS_CIPHER_AES_128_OFB, /**< AES 128-bit cipher in OFB mode. */ - MBEDTLS_CIPHER_AES_192_OFB, /**< AES 192-bit cipher in OFB mode. */ - MBEDTLS_CIPHER_AES_256_OFB, /**< AES 256-bit cipher in OFB mode. */ - MBEDTLS_CIPHER_AES_128_XTS, /**< AES 128-bit cipher in XTS block mode. */ - MBEDTLS_CIPHER_AES_256_XTS, /**< AES 256-bit cipher in XTS block mode. */ - MBEDTLS_CIPHER_CHACHA20, /**< ChaCha20 stream cipher. */ - MBEDTLS_CIPHER_CHACHA20_POLY1305, /**< ChaCha20-Poly1305 AEAD cipher. */ - MBEDTLS_CIPHER_AES_128_KW, /**< AES cipher with 128-bit NIST KW mode. */ - MBEDTLS_CIPHER_AES_192_KW, /**< AES cipher with 192-bit NIST KW mode. */ - MBEDTLS_CIPHER_AES_256_KW, /**< AES cipher with 256-bit NIST KW mode. */ - MBEDTLS_CIPHER_AES_128_KWP, /**< AES cipher with 128-bit NIST KWP mode. */ - MBEDTLS_CIPHER_AES_192_KWP, /**< AES cipher with 192-bit NIST KWP mode. */ - MBEDTLS_CIPHER_AES_256_KWP, /**< AES cipher with 256-bit NIST KWP mode. */ -} mbedtls_cipher_type_t; - -/** Supported cipher modes. */ -typedef enum { - MBEDTLS_MODE_NONE = 0, /**< None. */ - MBEDTLS_MODE_ECB, /**< The ECB cipher mode. */ - MBEDTLS_MODE_CBC, /**< The CBC cipher mode. */ - MBEDTLS_MODE_CFB, /**< The CFB cipher mode. */ - MBEDTLS_MODE_OFB, /**< The OFB cipher mode. */ - MBEDTLS_MODE_CTR, /**< The CTR cipher mode. */ - MBEDTLS_MODE_GCM, /**< The GCM cipher mode. */ - MBEDTLS_MODE_STREAM, /**< The stream cipher mode. */ - MBEDTLS_MODE_CCM, /**< The CCM cipher mode. */ - MBEDTLS_MODE_CCM_STAR_NO_TAG, /**< The CCM*-no-tag cipher mode. */ - MBEDTLS_MODE_XTS, /**< The XTS cipher mode. */ - MBEDTLS_MODE_CHACHAPOLY, /**< The ChaCha-Poly cipher mode. */ - MBEDTLS_MODE_KW, /**< The SP800-38F KW mode */ - MBEDTLS_MODE_KWP, /**< The SP800-38F KWP mode */ -} mbedtls_cipher_mode_t; - -/** Supported cipher padding types. */ -typedef enum { - MBEDTLS_PADDING_PKCS7 = 0, /**< PKCS7 padding (default). */ - MBEDTLS_PADDING_ONE_AND_ZEROS, /**< ISO/IEC 7816-4 padding. */ - MBEDTLS_PADDING_ZEROS_AND_LEN, /**< ANSI X.923 padding. */ - MBEDTLS_PADDING_ZEROS, /**< Zero padding (not reversible). */ - MBEDTLS_PADDING_NONE, /**< Never pad (full blocks only). */ -} mbedtls_cipher_padding_t; - -/** Type of operation. */ -typedef enum { - MBEDTLS_OPERATION_NONE = -1, - MBEDTLS_DECRYPT = 0, - MBEDTLS_ENCRYPT, -} mbedtls_operation_t; - -enum { - /** Undefined key length. */ - MBEDTLS_KEY_LENGTH_NONE = 0, - /** Key length, in bits (including parity), for DES keys. \warning DES is considered weak. */ - MBEDTLS_KEY_LENGTH_DES = 64, - /** Key length in bits, including parity, for DES in two-key EDE. \warning 3DES is considered weak. */ - MBEDTLS_KEY_LENGTH_DES_EDE = 128, - /** Key length in bits, including parity, for DES in three-key EDE. \warning 3DES is considered weak. */ - MBEDTLS_KEY_LENGTH_DES_EDE3 = 192, -}; - -/** Maximum length of any IV, in Bytes. */ -/* This should ideally be derived automatically from list of ciphers. - * This should be kept in sync with MBEDTLS_SSL_MAX_IV_LENGTH defined - * in library/ssl_misc.h. */ -#define MBEDTLS_MAX_IV_LENGTH 16 - -/** Maximum block size of any cipher, in Bytes. */ -/* This should ideally be derived automatically from list of ciphers. - * This should be kept in sync with MBEDTLS_SSL_MAX_BLOCK_LENGTH defined - * in library/ssl_misc.h. */ -#define MBEDTLS_MAX_BLOCK_LENGTH 16 - -/** Maximum key length, in Bytes. */ -/* This should ideally be derived automatically from list of ciphers. - * For now, only check whether XTS is enabled which uses 64 Byte keys, - * and use 32 Bytes as an upper bound for the maximum key length otherwise. - * This should be kept in sync with MBEDTLS_SSL_MAX_BLOCK_LENGTH defined - * in library/ssl_misc.h, which however deliberately ignores the case of XTS - * since the latter isn't used in SSL/TLS. */ -#if defined(MBEDTLS_CIPHER_MODE_XTS) -#define MBEDTLS_MAX_KEY_LENGTH 64 -#else -#define MBEDTLS_MAX_KEY_LENGTH 32 -#endif /* MBEDTLS_CIPHER_MODE_XTS */ - -/** - * Base cipher information (opaque struct). - */ -typedef struct mbedtls_cipher_base_t mbedtls_cipher_base_t; - -/** - * CMAC context (opaque struct). - */ -typedef struct mbedtls_cmac_context_t mbedtls_cmac_context_t; - -/** - * Cipher information. Allows calling cipher functions - * in a generic way. - * - * \note The library does not support custom cipher info structures, - * only built-in structures returned by the functions - * mbedtls_cipher_info_from_string(), - * mbedtls_cipher_info_from_type(), - * mbedtls_cipher_info_from_values(), - * mbedtls_cipher_info_from_psa(). - * - * \note Some fields store a value that has been right-shifted to save - * code-size, so should not be used directly. The accessor - * functions adjust for this and return the "natural" value. - */ -typedef struct mbedtls_cipher_info_t { - /** Name of the cipher. */ - const char *MBEDTLS_PRIVATE(name); - - /** The block size, in bytes. */ - unsigned int MBEDTLS_PRIVATE(block_size) : 5; - - /** IV or nonce size, in bytes (right shifted by #MBEDTLS_IV_SIZE_SHIFT). - * For ciphers that accept variable IV sizes, - * this is the recommended size. - */ - unsigned int MBEDTLS_PRIVATE(iv_size) : 3; - - /** The cipher key length, in bits (right shifted by #MBEDTLS_KEY_BITLEN_SHIFT). - * This is the default length for variable sized ciphers. - * Includes parity bits for ciphers like DES. - */ - unsigned int MBEDTLS_PRIVATE(key_bitlen) : 4; - - /** The cipher mode (as per mbedtls_cipher_mode_t). - * For example, MBEDTLS_MODE_CBC. - */ - unsigned int MBEDTLS_PRIVATE(mode) : 4; - - /** Full cipher identifier (as per mbedtls_cipher_type_t). - * For example, MBEDTLS_CIPHER_AES_256_CBC. - * - * This could be 7 bits, but 8 bits retains byte alignment for the - * next field, which reduces code size to access that field. - */ - unsigned int MBEDTLS_PRIVATE(type) : 8; - - /** Bitflag comprised of MBEDTLS_CIPHER_VARIABLE_IV_LEN and - * MBEDTLS_CIPHER_VARIABLE_KEY_LEN indicating whether the - * cipher supports variable IV or variable key sizes, respectively. - */ - unsigned int MBEDTLS_PRIVATE(flags) : 2; - - /** Index to LUT for base cipher information and functions. */ - unsigned int MBEDTLS_PRIVATE(base_idx) : 5; - -} mbedtls_cipher_info_t; - -/* For internal use only. - * These are used to more compactly represent the fields above. */ -#define MBEDTLS_KEY_BITLEN_SHIFT 6 -#define MBEDTLS_IV_SIZE_SHIFT 2 -/** - * Generic cipher context. - */ -typedef struct mbedtls_cipher_context_t { - /** Information about the associated cipher. */ - const mbedtls_cipher_info_t *MBEDTLS_PRIVATE(cipher_info); - - /** Key length to use. */ - int MBEDTLS_PRIVATE(key_bitlen); - - /** Operation that the key of the context has been - * initialized for. - */ - mbedtls_operation_t MBEDTLS_PRIVATE(operation); - -#if defined(MBEDTLS_CIPHER_MODE_WITH_PADDING) - /** Padding functions to use, if relevant for - * the specific cipher mode. - */ - void(*MBEDTLS_PRIVATE(add_padding))(unsigned char *output, size_t olen, - size_t data_len); - /* Report invalid-padding condition through the output parameter - * invalid_padding. To minimize changes in Mbed TLS 3.6, where this - * declaration is in a public header, use the public type size_t - * rather than the internal type mbedtls_ct_condition_t. */ - int(*MBEDTLS_PRIVATE(get_padding))(unsigned char *input, size_t ilen, - size_t *data_len, - size_t *invalid_padding); -#endif - - /** Buffer for input that has not been processed yet. */ - unsigned char MBEDTLS_PRIVATE(unprocessed_data)[MBEDTLS_MAX_BLOCK_LENGTH]; - - /** Number of Bytes that have not been processed yet. */ - size_t MBEDTLS_PRIVATE(unprocessed_len); - - /** Current IV or NONCE_COUNTER for CTR-mode, data unit (or sector) number - * for XTS-mode. */ - unsigned char MBEDTLS_PRIVATE(iv)[MBEDTLS_MAX_IV_LENGTH]; - - /** IV size in Bytes, for ciphers with variable-length IVs. */ - size_t MBEDTLS_PRIVATE(iv_size); - - /** The cipher-specific context. */ - void *MBEDTLS_PRIVATE(cipher_ctx); - -#if defined(MBEDTLS_CMAC_C) - /** CMAC-specific context. */ - mbedtls_cmac_context_t *MBEDTLS_PRIVATE(cmac_ctx); -#endif - -#if defined(MBEDTLS_USE_PSA_CRYPTO) && !defined(MBEDTLS_DEPRECATED_REMOVED) - /** Indicates whether the cipher operations should be performed - * by Mbed TLS' own crypto library or an external implementation - * of the PSA Crypto API. - * This is unset if the cipher context was established through - * mbedtls_cipher_setup(), and set if it was established through - * mbedtls_cipher_setup_psa(). - */ - unsigned char MBEDTLS_PRIVATE(psa_enabled); -#endif /* MBEDTLS_USE_PSA_CRYPTO && !MBEDTLS_DEPRECATED_REMOVED */ - -} mbedtls_cipher_context_t; - -/** - * \brief This function retrieves the list of ciphers supported - * by the generic cipher module. - * - * For any cipher identifier in the returned list, you can - * obtain the corresponding generic cipher information structure - * via mbedtls_cipher_info_from_type(), which can then be used - * to prepare a cipher context via mbedtls_cipher_setup(). - * - * - * \return A statically-allocated array of cipher identifiers - * of type cipher_type_t. The last entry is zero. - */ -const int *mbedtls_cipher_list(void); - -/** - * \brief This function retrieves the cipher-information - * structure associated with the given cipher name. - * - * \param cipher_name Name of the cipher to search for. This must not be - * \c NULL. - * - * \return The cipher information structure associated with the - * given \p cipher_name. - * \return \c NULL if the associated cipher information is not found. - */ -const mbedtls_cipher_info_t *mbedtls_cipher_info_from_string(const char *cipher_name); - -/** - * \brief This function retrieves the cipher-information - * structure associated with the given cipher type. - * - * \param cipher_type Type of the cipher to search for. - * - * \return The cipher information structure associated with the - * given \p cipher_type. - * \return \c NULL if the associated cipher information is not found. - */ -const mbedtls_cipher_info_t *mbedtls_cipher_info_from_type(const mbedtls_cipher_type_t cipher_type); - -/** - * \brief This function retrieves the cipher-information - * structure associated with the given cipher ID, - * key size and mode. - * - * \param cipher_id The ID of the cipher to search for. For example, - * #MBEDTLS_CIPHER_ID_AES. - * \param key_bitlen The length of the key in bits. - * \param mode The cipher mode. For example, #MBEDTLS_MODE_CBC. - * - * \return The cipher information structure associated with the - * given \p cipher_id. - * \return \c NULL if the associated cipher information is not found. - */ -const mbedtls_cipher_info_t *mbedtls_cipher_info_from_values(const mbedtls_cipher_id_t cipher_id, - int key_bitlen, - const mbedtls_cipher_mode_t mode); - -/** - * \brief Retrieve the identifier for a cipher info structure. - * - * \param[in] info The cipher info structure to query. - * This may be \c NULL. - * - * \return The full cipher identifier (\c MBEDTLS_CIPHER_xxx). - * \return #MBEDTLS_CIPHER_NONE if \p info is \c NULL. - */ -static inline mbedtls_cipher_type_t mbedtls_cipher_info_get_type( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return MBEDTLS_CIPHER_NONE; - } else { - return (mbedtls_cipher_type_t) info->MBEDTLS_PRIVATE(type); - } -} - -/** - * \brief Retrieve the operation mode for a cipher info structure. - * - * \param[in] info The cipher info structure to query. - * This may be \c NULL. - * - * \return The cipher mode (\c MBEDTLS_MODE_xxx). - * \return #MBEDTLS_MODE_NONE if \p info is \c NULL. - */ -static inline mbedtls_cipher_mode_t mbedtls_cipher_info_get_mode( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return MBEDTLS_MODE_NONE; - } else { - return (mbedtls_cipher_mode_t) info->MBEDTLS_PRIVATE(mode); - } -} - -/** - * \brief Retrieve the key size for a cipher info structure. - * - * \param[in] info The cipher info structure to query. - * This may be \c NULL. - * - * \return The key length in bits. - * For variable-sized ciphers, this is the default length. - * For DES, this includes the parity bits. - * \return \c 0 if \p info is \c NULL. - */ -static inline size_t mbedtls_cipher_info_get_key_bitlen( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return 0; - } else { - return ((size_t) info->MBEDTLS_PRIVATE(key_bitlen)) << MBEDTLS_KEY_BITLEN_SHIFT; - } -} - -/** - * \brief Retrieve the human-readable name for a - * cipher info structure. - * - * \param[in] info The cipher info structure to query. - * This may be \c NULL. - * - * \return The cipher name, which is a human readable string, - * with static storage duration. - * \return \c NULL if \p info is \c NULL. - */ -static inline const char *mbedtls_cipher_info_get_name( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return NULL; - } else { - return info->MBEDTLS_PRIVATE(name); - } -} - -/** - * \brief This function returns the size of the IV or nonce - * for the cipher info structure, in bytes. - * - * \param info The cipher info structure. This may be \c NULL. - * - * \return The recommended IV size. - * \return \c 0 for ciphers not using an IV or a nonce. - * \return \c 0 if \p info is \c NULL. - */ -static inline size_t mbedtls_cipher_info_get_iv_size( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return 0; - } - - return ((size_t) info->MBEDTLS_PRIVATE(iv_size)) << MBEDTLS_IV_SIZE_SHIFT; -} - -/** - * \brief This function returns the block size of the given - * cipher info structure in bytes. - * - * \param info The cipher info structure. This may be \c NULL. - * - * \return The block size of the cipher. - * \return \c 1 if the cipher is a stream cipher. - * \return \c 0 if \p info is \c NULL. - */ -static inline size_t mbedtls_cipher_info_get_block_size( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return 0; - } - - return (size_t) (info->MBEDTLS_PRIVATE(block_size)); -} - -/** - * \brief This function returns a non-zero value if the key length for - * the given cipher is variable. - * - * \param info The cipher info structure. This may be \c NULL. - * - * \return Non-zero if the key length is variable, \c 0 otherwise. - * \return \c 0 if the given pointer is \c NULL. - */ -static inline int mbedtls_cipher_info_has_variable_key_bitlen( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return 0; - } - - return info->MBEDTLS_PRIVATE(flags) & MBEDTLS_CIPHER_VARIABLE_KEY_LEN; -} - -/** - * \brief This function returns a non-zero value if the IV size for - * the given cipher is variable. - * - * \param info The cipher info structure. This may be \c NULL. - * - * \return Non-zero if the IV size is variable, \c 0 otherwise. - * \return \c 0 if the given pointer is \c NULL. - */ -static inline int mbedtls_cipher_info_has_variable_iv_size( - const mbedtls_cipher_info_t *info) -{ - if (info == NULL) { - return 0; - } - - return info->MBEDTLS_PRIVATE(flags) & MBEDTLS_CIPHER_VARIABLE_IV_LEN; -} - -/** - * \brief This function initializes a \p ctx as NONE. - * - * \param ctx The context to be initialized. This must not be \c NULL. - */ -void mbedtls_cipher_init(mbedtls_cipher_context_t *ctx); - -/** - * \brief This function frees and clears the cipher-specific - * context of \p ctx. Freeing \p ctx itself remains the - * responsibility of the caller. - * - * \param ctx The context to be freed. If this is \c NULL, the - * function has no effect, otherwise this must point to an - * initialized context. - */ -void mbedtls_cipher_free(mbedtls_cipher_context_t *ctx); - - -/** - * \brief This function prepares a cipher context for - * use with the given cipher primitive. - * - * \note After calling this function, you should call - * mbedtls_cipher_setkey() and, if the mode uses padding, - * mbedtls_cipher_set_padding_mode(), then for each - * message to encrypt or decrypt with this key, either: - * - mbedtls_cipher_crypt() for one-shot processing with - * non-AEAD modes; - * - mbedtls_cipher_auth_encrypt_ext() or - * mbedtls_cipher_auth_decrypt_ext() for one-shot - * processing with AEAD modes or NIST_KW; - * - for multi-part processing, see the documentation of - * mbedtls_cipher_reset(). - * - * \param ctx The context to prepare. This must be initialized by - * a call to mbedtls_cipher_init() first. - * \param cipher_info The cipher to use. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_ALLOC_FAILED if allocation of the - * cipher-specific context fails. - */ -int mbedtls_cipher_setup(mbedtls_cipher_context_t *ctx, - const mbedtls_cipher_info_t *cipher_info); - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief This function initializes a cipher context for - * PSA-based use with the given cipher primitive. - * - * \deprecated This function is deprecated and will be removed in a - * future version of the library. - * Please use psa_aead_xxx() / psa_cipher_xxx() directly - * instead. - * - * \note See #MBEDTLS_USE_PSA_CRYPTO for information on PSA. - * - * \param ctx The context to initialize. May not be \c NULL. - * \param cipher_info The cipher to use. - * \param taglen For AEAD ciphers, the length in bytes of the - * authentication tag to use. Subsequent uses of - * mbedtls_cipher_auth_encrypt_ext() or - * mbedtls_cipher_auth_decrypt_ext() must provide - * the same tag length. - * For non-AEAD ciphers, the value must be \c 0. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_ALLOC_FAILED if allocation of the - * cipher-specific context fails. - */ -int MBEDTLS_DEPRECATED mbedtls_cipher_setup_psa(mbedtls_cipher_context_t *ctx, - const mbedtls_cipher_info_t *cipher_info, - size_t taglen); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -/** - * \brief This function returns the block size of the given cipher - * in bytes. - * - * \param ctx The context of the cipher. - * - * \return The block size of the underlying cipher. - * \return \c 1 if the cipher is a stream cipher. - * \return \c 0 if \p ctx has not been initialized. - */ -static inline unsigned int mbedtls_cipher_get_block_size( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return 0; - } - - return (unsigned int) ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(block_size); -} - -/** - * \brief This function returns the mode of operation for - * the cipher. For example, MBEDTLS_MODE_CBC. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The mode of operation. - * \return #MBEDTLS_MODE_NONE if \p ctx has not been initialized. - */ -static inline mbedtls_cipher_mode_t mbedtls_cipher_get_cipher_mode( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return MBEDTLS_MODE_NONE; - } - - return (mbedtls_cipher_mode_t) ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(mode); -} - -/** - * \brief This function returns the size of the IV or nonce - * of the cipher, in Bytes. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The recommended IV size if no IV has been set. - * \return \c 0 for ciphers not using an IV or a nonce. - * \return The actual size if an IV has been set. - */ -static inline int mbedtls_cipher_get_iv_size( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return 0; - } - - if (ctx->MBEDTLS_PRIVATE(iv_size) != 0) { - return (int) ctx->MBEDTLS_PRIVATE(iv_size); - } - - return (int) (((int) ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(iv_size)) << - MBEDTLS_IV_SIZE_SHIFT); -} - -/** - * \brief This function returns the type of the given cipher. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The type of the cipher. - * \return #MBEDTLS_CIPHER_NONE if \p ctx has not been initialized. - */ -static inline mbedtls_cipher_type_t mbedtls_cipher_get_type( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return MBEDTLS_CIPHER_NONE; - } - - return (mbedtls_cipher_type_t) ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(type); -} - -/** - * \brief This function returns the name of the given cipher - * as a string. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The name of the cipher. - * \return NULL if \p ctx has not been not initialized. - */ -static inline const char *mbedtls_cipher_get_name( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return 0; - } - - return ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(name); -} - -/** - * \brief This function returns the key length of the cipher. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The key length of the cipher in bits. - * \return #MBEDTLS_KEY_LENGTH_NONE if \p ctx has not been - * initialized. - */ -static inline int mbedtls_cipher_get_key_bitlen( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return MBEDTLS_KEY_LENGTH_NONE; - } - - return (int) ctx->MBEDTLS_PRIVATE(cipher_info)->MBEDTLS_PRIVATE(key_bitlen) << - MBEDTLS_KEY_BITLEN_SHIFT; -} - -/** - * \brief This function returns the operation of the given cipher. - * - * \param ctx The context of the cipher. This must be initialized. - * - * \return The type of operation: #MBEDTLS_ENCRYPT or #MBEDTLS_DECRYPT. - * \return #MBEDTLS_OPERATION_NONE if \p ctx has not been initialized. - */ -static inline mbedtls_operation_t mbedtls_cipher_get_operation( - const mbedtls_cipher_context_t *ctx) -{ - if (ctx->MBEDTLS_PRIVATE(cipher_info) == NULL) { - return MBEDTLS_OPERATION_NONE; - } - - return ctx->MBEDTLS_PRIVATE(operation); -} - -/** - * \brief This function sets the key to use with the given context. - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a cipher information structure. - * \param key The key to use. This must be a readable buffer of at - * least \p key_bitlen Bits. - * \param key_bitlen The key length to use, in Bits. - * \param operation The operation that the key will be used for: - * #MBEDTLS_ENCRYPT or #MBEDTLS_DECRYPT. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_setkey(mbedtls_cipher_context_t *ctx, - const unsigned char *key, - int key_bitlen, - const mbedtls_operation_t operation); - -#if defined(MBEDTLS_CIPHER_MODE_WITH_PADDING) -/** - * \brief This function sets the padding mode, for cipher modes - * that use padding. - * - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a cipher information structure. - * \param mode The padding mode. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE - * if the selected padding mode is not supported. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA if the cipher mode - * does not support padding. - */ -int mbedtls_cipher_set_padding_mode(mbedtls_cipher_context_t *ctx, - mbedtls_cipher_padding_t mode); -#endif /* MBEDTLS_CIPHER_MODE_WITH_PADDING */ - -/** - * \brief This function sets the initialization vector (IV) - * or nonce. - * - * \note Some ciphers do not use IVs nor nonce. For these - * ciphers, this function has no effect. - * - * \note For #MBEDTLS_CIPHER_CHACHA20, the nonce length must - * be 12, and the initial counter value is 0. - * - * \note For #MBEDTLS_CIPHER_CHACHA20_POLY1305, the nonce length - * must be 12. - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a cipher information structure. - * \param iv The IV to use, or NONCE_COUNTER for CTR-mode ciphers. This - * must be a readable buffer of at least \p iv_len Bytes. - * \param iv_len The IV length for ciphers with variable-size IV. - * This parameter is discarded by ciphers with fixed-size IV. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - */ -int mbedtls_cipher_set_iv(mbedtls_cipher_context_t *ctx, - const unsigned char *iv, - size_t iv_len); - -/** - * \brief This function resets the cipher state. - * - * \note With non-AEAD ciphers, the order of calls for each message - * is as follows: - * 1. mbedtls_cipher_set_iv() if the mode uses an IV/nonce; - * 2. mbedtls_cipher_reset(); - * 3. mbedtls_cipher_update() zero, one or more times; - * 4. mbedtls_cipher_finish_padded() (recommended for decryption - * if the mode uses padding) or mbedtls_cipher_finish(). - * . - * This sequence can be repeated to encrypt or decrypt multiple - * messages with the same key. - * - * \note With AEAD ciphers, the order of calls for each message - * is as follows: - * 1. mbedtls_cipher_set_iv() if the mode uses an IV/nonce; - * 2. mbedtls_cipher_reset(); - * 3. mbedtls_cipher_update_ad(); - * 4. mbedtls_cipher_update() zero, one or more times; - * 5. mbedtls_cipher_finish() (or mbedtls_cipher_finish_padded()); - * 6. mbedtls_cipher_check_tag() (for decryption) or - * mbedtls_cipher_write_tag() (for encryption). - * . - * This sequence can be repeated to encrypt or decrypt multiple - * messages with the same key. - * - * \param ctx The generic cipher context. This must be bound to a key. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - */ -int mbedtls_cipher_reset(mbedtls_cipher_context_t *ctx); - -#if defined(MBEDTLS_GCM_C) || defined(MBEDTLS_CHACHAPOLY_C) -/** - * \brief This function adds additional data for AEAD ciphers. - * Currently supported with GCM and ChaCha20+Poly1305. - * - * \param ctx The generic cipher context. This must be initialized. - * \param ad The additional data to use. This must be a readable - * buffer of at least \p ad_len Bytes. - * \param ad_len The length of \p ad in Bytes. - * - * \return \c 0 on success. - * \return A specific error code on failure. - */ -int mbedtls_cipher_update_ad(mbedtls_cipher_context_t *ctx, - const unsigned char *ad, size_t ad_len); -#endif /* MBEDTLS_GCM_C || MBEDTLS_CHACHAPOLY_C */ - -/** - * \brief The generic cipher update function. It encrypts or - * decrypts using the given cipher context. Writes as - * many block-sized blocks of data as possible to output. - * Any data that cannot be written immediately is either - * added to the next block, or flushed when - * mbedtls_cipher_finish() or mbedtls_cipher_finish_padded() - * is called. - * Exception: For MBEDTLS_MODE_ECB, expects a single block - * in size. For example, 16 Bytes for AES. - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a key. - * \param input The buffer holding the input data. This must be a - * readable buffer of at least \p ilen Bytes. - * \param ilen The length of the input data. - * \param output The buffer for the output data. This must be able to - * hold at least `ilen + block_size`. This must not be the - * same buffer as \p input. - * \param olen The length of the output data, to be updated with the - * actual number of Bytes written. This must not be - * \c NULL. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE on an - * unsupported mode for a cipher. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_update(mbedtls_cipher_context_t *ctx, - const unsigned char *input, - size_t ilen, unsigned char *output, - size_t *olen); - -/** - * \brief The generic cipher finalization function. If data still - * needs to be flushed from an incomplete block, the data - * contained in it is padded to the size of - * the last block, and written to the \p output buffer. - * - * \warning This function reports invalid padding through an error - * code. Adversaries may be able to decrypt encrypted - * data if they can submit chosen ciphertexts and - * detect whether it has valid padding or not, - * either through direct observation or through a side - * channel such as timing. This is known as a - * padding oracle attack. - * Therefore applications that call this function for - * decryption with a cipher that involves padding - * should take care around error handling. Preferably, - * such applications should use - * mbedtls_cipher_finish_padded() instead of this function. - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a key. - * \param output The buffer to write data to. This needs to be a writable - * buffer of at least block_size Bytes. - * \param olen The length of the data written to the \p output buffer. - * This may not be \c NULL. - * Note that when decrypting in a mode with padding, - * the actual output length is sensitive and may be - * used to mount a padding oracle attack (see warning - * above), although less efficiently than through - * the invalid-padding condition. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED on decryption - * expecting a full block but not receiving one. - * \return #MBEDTLS_ERR_CIPHER_INVALID_PADDING on invalid padding - * while decrypting. Note that invalid-padding errors - * should be handled carefully; see the warning above. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_finish(mbedtls_cipher_context_t *ctx, - unsigned char *output, size_t *olen); - -/** - * \brief The generic cipher finalization function. If data still - * needs to be flushed from an incomplete block, the data - * contained in it is padded to the size of - * the last block, and written to the \p output buffer. - * - * \note This function is similar to mbedtls_cipher_finish(). - * The only difference is that it reports invalid padding - * decryption differently, through the \p invalid_padding - * parameter rather than an error code. - * For encryption, and in modes without padding (including - * all authenticated modes), this function is identical - * to mbedtls_cipher_finish(). - * - * \param[in,out] ctx The generic cipher context. This must be initialized and - * bound to a key. - * \param[out] output The buffer to write data to. This needs to be a writable - * buffer of at least block_size Bytes. - * \param[out] olen The length of the data written to the \p output buffer. - * This may not be \c NULL. - * Note that when decrypting in a mode with padding, - * the actual output length is sensitive and may be - * used to mount a padding oracle attack (see warning - * on mbedtls_cipher_finish()). - * \param[out] invalid_padding - * If this function returns \c 0 on decryption, - * \p *invalid_padding is \c 0 if the ciphertext was - * valid, and all-bits-one if the ciphertext had invalid - * padding. - * On encryption, or in a mode without padding (including - * all authenticated modes), \p *invalid_padding is \c 0 - * on success. - * The value in \p *invalid_padding is unspecified if - * this function returns a nonzero status. - * - * \return \c 0 on success. - * Also \c 0 for decryption with invalid padding. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED on decryption - * expecting a full block but not receiving one. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_finish_padded(mbedtls_cipher_context_t *ctx, - unsigned char *output, size_t *olen, - size_t *invalid_padding); - -#if defined(MBEDTLS_GCM_C) || defined(MBEDTLS_CHACHAPOLY_C) -/** - * \brief This function writes a tag for AEAD ciphers. - * Currently supported with GCM and ChaCha20+Poly1305. - * This must be called after mbedtls_cipher_finish() - * or mbedtls_cipher_finish_padded(). - * - * \param ctx The generic cipher context. This must be initialized, - * bound to a key, and have just completed a cipher - * operation through mbedtls_cipher_finish() the tag for - * which should be written. - * \param tag The buffer to write the tag to. This must be a writable - * buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to write. - * - * \return \c 0 on success. - * \return A specific error code on failure. - */ -int mbedtls_cipher_write_tag(mbedtls_cipher_context_t *ctx, - unsigned char *tag, size_t tag_len); - -/** - * \brief This function checks the tag for AEAD ciphers. - * Currently supported with GCM and ChaCha20+Poly1305. - * This must be called after mbedtls_cipher_finish() - * or mbedtls_cipher_finish_padded(). - * - * \param ctx The generic cipher context. This must be initialized. - * \param tag The buffer holding the tag. This must be a readable - * buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to check. - * - * \return \c 0 on success. - * \return A specific error code on failure. - */ -int mbedtls_cipher_check_tag(mbedtls_cipher_context_t *ctx, - const unsigned char *tag, size_t tag_len); -#endif /* MBEDTLS_GCM_C || MBEDTLS_CHACHAPOLY_C */ - -/** - * \brief The generic all-in-one encryption/decryption function, - * for all ciphers except AEAD constructs. - * - * \param ctx The generic cipher context. This must be initialized. - * \param iv The IV to use, or NONCE_COUNTER for CTR-mode ciphers. - * This must be a readable buffer of at least \p iv_len - * Bytes. - * \param iv_len The IV length for ciphers with variable-size IV. - * This parameter is discarded by ciphers with fixed-size - * IV. - * \param input The buffer holding the input data. This must be a - * readable buffer of at least \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * \param output The buffer for the output data. This must be able to - * hold at least `ilen + block_size`. This must not be the - * same buffer as \p input. - * \param olen The length of the output data, to be updated with the - * actual number of Bytes written. This must not be - * \c NULL. - * - * \note Some ciphers do not use IVs nor nonce. For these - * ciphers, use \p iv = NULL and \p iv_len = 0. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_FULL_BLOCK_EXPECTED on decryption - * expecting a full block but not receiving one. - * \return #MBEDTLS_ERR_CIPHER_INVALID_PADDING on invalid padding - * while decrypting. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_crypt(mbedtls_cipher_context_t *ctx, - const unsigned char *iv, size_t iv_len, - const unsigned char *input, size_t ilen, - unsigned char *output, size_t *olen); - -#if defined(MBEDTLS_CIPHER_MODE_AEAD) || defined(MBEDTLS_NIST_KW_C) -/** - * \brief The authenticated encryption (AEAD/NIST_KW) function. - * - * \note For AEAD modes, the tag will be appended to the - * ciphertext, as recommended by RFC 5116. - * (NIST_KW doesn't have a separate tag.) - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a key, with an AEAD algorithm or NIST_KW. - * \param iv The nonce to use. This must be a readable buffer of - * at least \p iv_len Bytes and may be \c NULL if \p - * iv_len is \c 0. - * \param iv_len The length of the nonce. For AEAD ciphers, this must - * satisfy the constraints imposed by the cipher used. - * For NIST_KW, this must be \c 0. - * \param ad The additional data to authenticate. This must be a - * readable buffer of at least \p ad_len Bytes, and may - * be \c NULL is \p ad_len is \c 0. - * \param ad_len The length of \p ad. For NIST_KW, this must be \c 0. - * \param input The buffer holding the input data. This must be a - * readable buffer of at least \p ilen Bytes, and may be - * \c NULL if \p ilen is \c 0. - * \param ilen The length of the input data. - * \param output The buffer for the output data. This must be a - * writable buffer of at least \p output_len Bytes, and - * must not be \c NULL. - * \param output_len The length of the \p output buffer in Bytes. For AEAD - * ciphers, this must be at least \p ilen + \p tag_len. - * For NIST_KW, this must be at least \p ilen + 8 - * (rounded up to a multiple of 8 if KWP is used); - * \p ilen + 15 is always a safe value. - * \param olen This will be filled with the actual number of Bytes - * written to the \p output buffer. This must point to a - * writable object of type \c size_t. - * \param tag_len The desired length of the authentication tag. For AEAD - * ciphers, this must match the constraints imposed by - * the cipher used, and in particular must not be \c 0. - * For NIST_KW, this must be \c 0. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_auth_encrypt_ext(mbedtls_cipher_context_t *ctx, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, size_t ilen, - unsigned char *output, size_t output_len, - size_t *olen, size_t tag_len); - -/** - * \brief The authenticated encryption (AEAD/NIST_KW) function. - * - * \note If the data is not authentic, then the output buffer - * is zeroed out to prevent the unauthentic plaintext being - * used, making this interface safer. - * - * \note For AEAD modes, the tag must be appended to the - * ciphertext, as recommended by RFC 5116. - * (NIST_KW doesn't have a separate tag.) - * - * \param ctx The generic cipher context. This must be initialized and - * bound to a key, with an AEAD algorithm or NIST_KW. - * \param iv The nonce to use. This must be a readable buffer of - * at least \p iv_len Bytes and may be \c NULL if \p - * iv_len is \c 0. - * \param iv_len The length of the nonce. For AEAD ciphers, this must - * satisfy the constraints imposed by the cipher used. - * For NIST_KW, this must be \c 0. - * \param ad The additional data to authenticate. This must be a - * readable buffer of at least \p ad_len Bytes, and may - * be \c NULL is \p ad_len is \c 0. - * \param ad_len The length of \p ad. For NIST_KW, this must be \c 0. - * \param input The buffer holding the input data. This must be a - * readable buffer of at least \p ilen Bytes, and may be - * \c NULL if \p ilen is \c 0. - * \param ilen The length of the input data. For AEAD ciphers this - * must be at least \p tag_len. For NIST_KW this must be - * at least \c 8. - * \param output The buffer for the output data. This must be a - * writable buffer of at least \p output_len Bytes, and - * may be \c NULL if \p output_len is \c 0. - * \param output_len The length of the \p output buffer in Bytes. For AEAD - * ciphers, this must be at least \p ilen - \p tag_len. - * For NIST_KW, this must be at least \p ilen - 8. - * \param olen This will be filled with the actual number of Bytes - * written to the \p output buffer. This must point to a - * writable object of type \c size_t. - * \param tag_len The actual length of the authentication tag. For AEAD - * ciphers, this must match the constraints imposed by - * the cipher used, and in particular must not be \c 0. - * For NIST_KW, this must be \c 0. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA on - * parameter-verification failure. - * \return #MBEDTLS_ERR_CIPHER_AUTH_FAILED if data is not authentic. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_auth_decrypt_ext(mbedtls_cipher_context_t *ctx, - const unsigned char *iv, size_t iv_len, - const unsigned char *ad, size_t ad_len, - const unsigned char *input, size_t ilen, - unsigned char *output, size_t output_len, - size_t *olen, size_t tag_len); -#endif /* MBEDTLS_CIPHER_MODE_AEAD || MBEDTLS_NIST_KW_C */ -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_CIPHER_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/cmac.h b/android/mbedtls/arm64-v8a/include/mbedtls/cmac.h deleted file mode 100644 index 97b86fc42..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/cmac.h +++ /dev/null @@ -1,246 +0,0 @@ -/** - * \file cmac.h - * - * \brief This file contains CMAC definitions and functions. - * - * The Cipher-based Message Authentication Code (CMAC) Mode for - * Authentication is defined in RFC-4493: The AES-CMAC Algorithm. - * It is supported with AES and DES. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CMAC_H -#define MBEDTLS_CMAC_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/cipher.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MBEDTLS_AES_BLOCK_SIZE 16 -#define MBEDTLS_DES3_BLOCK_SIZE 8 - -/* We don't support Camellia or ARIA in this module */ -#if defined(MBEDTLS_AES_C) -#define MBEDTLS_CMAC_MAX_BLOCK_SIZE 16 /**< The longest block used by CMAC is that of AES. */ -#else -#define MBEDTLS_CMAC_MAX_BLOCK_SIZE 8 /**< The longest block used by CMAC is that of 3DES. */ -#endif - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** The longest block supported by the cipher module. - * - * \deprecated - * For the maximum block size of a cipher supported by the CMAC module, - * use #MBEDTLS_CMAC_MAX_BLOCK_SIZE. - * For the maximum block size of a cipher supported by the cipher module, - * use #MBEDTLS_MAX_BLOCK_LENGTH. - */ -/* Before Mbed TLS 3.5, this was the maximum block size supported by the CMAC - * module, so it didn't take Camellia or ARIA into account. Since the name - * of the macro doesn't even convey "CMAC", this was misleading. Now the size - * is sufficient for any cipher, but the name is defined in cmac.h for - * backward compatibility. */ -#define MBEDTLS_CIPHER_BLKSIZE_MAX MBEDTLS_MAX_BLOCK_LENGTH -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -#if !defined(MBEDTLS_CMAC_ALT) - -/** - * The CMAC context structure. - */ -struct mbedtls_cmac_context_t { - /** The internal state of the CMAC algorithm. */ - unsigned char MBEDTLS_PRIVATE(state)[MBEDTLS_CMAC_MAX_BLOCK_SIZE]; - - /** Unprocessed data - either data that was not block aligned and is still - * pending processing, or the final block. */ - unsigned char MBEDTLS_PRIVATE(unprocessed_block)[MBEDTLS_CMAC_MAX_BLOCK_SIZE]; - - /** The length of data pending processing. */ - size_t MBEDTLS_PRIVATE(unprocessed_len); -}; - -#else /* !MBEDTLS_CMAC_ALT */ -#include "cmac_alt.h" -#endif /* !MBEDTLS_CMAC_ALT */ - -/** - * \brief This function starts a new CMAC computation - * by setting the CMAC key, and preparing to authenticate - * the input data. - * It must be called with an initialized cipher context. - * - * Once this function has completed, data can be supplied - * to the CMAC computation by calling - * mbedtls_cipher_cmac_update(). - * - * To start a CMAC computation using the same key as a previous - * CMAC computation, use mbedtls_cipher_cmac_finish(). - * - * \note When the CMAC implementation is supplied by an alternate - * implementation (through #MBEDTLS_CMAC_ALT), some ciphers - * may not be supported by that implementation, and thus - * return an error. Alternate implementations must support - * AES-128 and AES-256, and may support AES-192 and 3DES. - * - * \param ctx The cipher context used for the CMAC operation, initialized - * as one of the following types: MBEDTLS_CIPHER_AES_128_ECB, - * MBEDTLS_CIPHER_AES_192_ECB, MBEDTLS_CIPHER_AES_256_ECB, - * or MBEDTLS_CIPHER_DES_EDE3_ECB. - * \param key The CMAC key. - * \param keybits The length of the CMAC key in bits. - * Must be supported by the cipher. - * - * \return \c 0 on success. - * \return A cipher-specific error code on failure. - */ -int mbedtls_cipher_cmac_starts(mbedtls_cipher_context_t *ctx, - const unsigned char *key, size_t keybits); - -/** - * \brief This function feeds an input buffer into an ongoing CMAC - * computation. - * - * The CMAC computation must have previously been started - * by calling mbedtls_cipher_cmac_starts() or - * mbedtls_cipher_cmac_reset(). - * - * Call this function as many times as needed to input the - * data to be authenticated. - * Once all of the required data has been input, - * call mbedtls_cipher_cmac_finish() to obtain the result - * of the CMAC operation. - * - * \param ctx The cipher context used for the CMAC operation. - * \param input The buffer holding the input data. - * \param ilen The length of the input data. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA - * if parameter verification fails. - */ -int mbedtls_cipher_cmac_update(mbedtls_cipher_context_t *ctx, - const unsigned char *input, size_t ilen); - -/** - * \brief This function finishes an ongoing CMAC operation, and - * writes the result to the output buffer. - * - * It should be followed either by - * mbedtls_cipher_cmac_reset(), which starts another CMAC - * operation with the same key, or mbedtls_cipher_free(), - * which clears the cipher context. - * - * \param ctx The cipher context used for the CMAC operation. - * \param output The output buffer for the CMAC checksum result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA - * if parameter verification fails. - */ -int mbedtls_cipher_cmac_finish(mbedtls_cipher_context_t *ctx, - unsigned char *output); - -/** - * \brief This function starts a new CMAC operation with the same - * key as the previous one. - * - * It should be called after finishing the previous CMAC - * operation with mbedtls_cipher_cmac_finish(). - * After calling this function, - * call mbedtls_cipher_cmac_update() to supply the new - * CMAC operation with data. - * - * \param ctx The cipher context used for the CMAC operation. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA - * if parameter verification fails. - */ -int mbedtls_cipher_cmac_reset(mbedtls_cipher_context_t *ctx); - -/** - * \brief This function calculates the full generic CMAC - * on the input buffer with the provided key. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The CMAC result is calculated as - * output = generic CMAC(cmac key, input buffer). - * - * \note When the CMAC implementation is supplied by an alternate - * implementation (through #MBEDTLS_CMAC_ALT), some ciphers - * may not be supported by that implementation, and thus - * return an error. Alternate implementations must support - * AES-128 and AES-256, and may support AES-192 and 3DES. - * - * \param cipher_info The cipher information. - * \param key The CMAC key. - * \param keylen The length of the CMAC key in bits. - * \param input The buffer holding the input data. - * \param ilen The length of the input data. - * \param output The buffer for the generic CMAC result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA - * if parameter verification fails. - */ -int mbedtls_cipher_cmac(const mbedtls_cipher_info_t *cipher_info, - const unsigned char *key, size_t keylen, - const unsigned char *input, size_t ilen, - unsigned char *output); - -#if defined(MBEDTLS_AES_C) -/** - * \brief This function implements the AES-CMAC-PRF-128 pseudorandom - * function, as defined in - * RFC-4615: The Advanced Encryption Standard-Cipher-based - * Message Authentication Code-Pseudo-Random Function-128 - * (AES-CMAC-PRF-128) Algorithm for the Internet Key - * Exchange Protocol (IKE). - * - * \param key The key to use. - * \param key_len The key length in Bytes. - * \param input The buffer holding the input data. - * \param in_len The length of the input data in Bytes. - * \param output The buffer holding the generated 16 Bytes of - * pseudorandom output. - * - * \return \c 0 on success. - */ -int mbedtls_aes_cmac_prf_128(const unsigned char *key, size_t key_len, - const unsigned char *input, size_t in_len, - unsigned char output[16]); -#endif /* MBEDTLS_AES_C */ - -#if defined(MBEDTLS_SELF_TEST) && (defined(MBEDTLS_AES_C) || defined(MBEDTLS_DES_C)) -/** - * \brief The CMAC checkup routine. - * - * \note In case the CMAC routines are provided by an alternative - * implementation (i.e. #MBEDTLS_CMAC_ALT is defined), the - * checkup routine will succeed even if the implementation does - * not support the less widely used AES-192 or 3DES primitives. - * The self-test requires at least AES-128 and AES-256 to be - * supported by the underlying implementation. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_cmac_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST && ( MBEDTLS_AES_C || MBEDTLS_DES_C ) */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_CMAC_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/compat-2.x.h b/android/mbedtls/arm64-v8a/include/mbedtls/compat-2.x.h deleted file mode 100644 index 096341ba7..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/compat-2.x.h +++ /dev/null @@ -1,46 +0,0 @@ -/** - * \file compat-2.x.h - * - * \brief Compatibility definitions - * - * \deprecated Use the new names directly instead - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#if defined(MBEDTLS_DEPRECATED_WARNING) -#warning "Including compat-2.x.h is deprecated" -#endif - -#ifndef MBEDTLS_COMPAT2X_H -#define MBEDTLS_COMPAT2X_H - -/* - * Macros for renamed functions - */ -#define mbedtls_ctr_drbg_update_ret mbedtls_ctr_drbg_update -#define mbedtls_hmac_drbg_update_ret mbedtls_hmac_drbg_update -#define mbedtls_md5_starts_ret mbedtls_md5_starts -#define mbedtls_md5_update_ret mbedtls_md5_update -#define mbedtls_md5_finish_ret mbedtls_md5_finish -#define mbedtls_md5_ret mbedtls_md5 -#define mbedtls_ripemd160_starts_ret mbedtls_ripemd160_starts -#define mbedtls_ripemd160_update_ret mbedtls_ripemd160_update -#define mbedtls_ripemd160_finish_ret mbedtls_ripemd160_finish -#define mbedtls_ripemd160_ret mbedtls_ripemd160 -#define mbedtls_sha1_starts_ret mbedtls_sha1_starts -#define mbedtls_sha1_update_ret mbedtls_sha1_update -#define mbedtls_sha1_finish_ret mbedtls_sha1_finish -#define mbedtls_sha1_ret mbedtls_sha1 -#define mbedtls_sha256_starts_ret mbedtls_sha256_starts -#define mbedtls_sha256_update_ret mbedtls_sha256_update -#define mbedtls_sha256_finish_ret mbedtls_sha256_finish -#define mbedtls_sha256_ret mbedtls_sha256 -#define mbedtls_sha512_starts_ret mbedtls_sha512_starts -#define mbedtls_sha512_update_ret mbedtls_sha512_update -#define mbedtls_sha512_finish_ret mbedtls_sha512_finish -#define mbedtls_sha512_ret mbedtls_sha512 - -#endif /* MBEDTLS_COMPAT2X_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_crypto.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_crypto.h deleted file mode 100644 index 331ac9b2d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_crypto.h +++ /dev/null @@ -1,535 +0,0 @@ -/** - * \file mbedtls/config_adjust_legacy_crypto.h - * \brief Adjust legacy configuration configuration - * - * This is an internal header. Do not include it directly. - * - * Automatically enable certain dependencies. Generally, MBEDTLS_xxx - * configurations need to be explicitly enabled by the user: enabling - * MBEDTLS_xxx_A but not MBEDTLS_xxx_B when A requires B results in a - * compilation error. However, we do automatically enable certain options - * in some circumstances. One case is if MBEDTLS_xxx_B is an internal option - * used to identify parts of a module that are used by other module, and we - * don't want to make the symbol MBEDTLS_xxx_B part of the public API. - * Another case is if A didn't depend on B in earlier versions, and we - * want to use B in A but we need to preserve backward compatibility with - * configurations that explicitly activate MBEDTLS_xxx_A but not - * MBEDTLS_xxx_B. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_LEGACY_CRYPTO_H -#define MBEDTLS_CONFIG_ADJUST_LEGACY_CRYPTO_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/* Ideally, we'd set those as defaults in mbedtls_config.h, but - * putting an #ifdef _WIN32 in mbedtls_config.h would confuse config.py. - * - * So, adjust it here. - * Not related to crypto, but this is the bottom of the stack. */ -#if defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER <= 1900) -#if !defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) && \ - !defined(MBEDTLS_PLATFORM_SNPRINTF_MACRO) -#define MBEDTLS_PLATFORM_SNPRINTF_ALT -#endif -#if !defined(MBEDTLS_PLATFORM_VSNPRINTF_ALT) && \ - !defined(MBEDTLS_PLATFORM_VSNPRINTF_MACRO) -#define MBEDTLS_PLATFORM_VSNPRINTF_ALT -#endif -#endif /* _MINGW32__ || (_MSC_VER && (_MSC_VER <= 1900)) */ - -/* If MBEDTLS_PSA_CRYPTO_C is defined, make sure MBEDTLS_PSA_CRYPTO_CLIENT - * is defined as well to include all PSA code. - */ -#if defined(MBEDTLS_PSA_CRYPTO_C) -#define MBEDTLS_PSA_CRYPTO_CLIENT -#endif /* MBEDTLS_PSA_CRYPTO_C */ - -/* Auto-enable CIPHER_C when any of the unauthenticated ciphers is builtin - * in PSA. */ -#if defined(MBEDTLS_PSA_CRYPTO_C) && \ - (defined(MBEDTLS_PSA_BUILTIN_ALG_STREAM_CIPHER) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CTR) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CFB) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_OFB) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_ECB_NO_PADDING) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CBC_NO_PADDING) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CBC_PKCS7) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CCM_STAR_NO_TAG) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CMAC)) -#define MBEDTLS_CIPHER_C -#endif - -/* Auto-enable MBEDTLS_MD_LIGHT based on MBEDTLS_MD_C. - * This allows checking for MD_LIGHT rather than MD_LIGHT || MD_C. - */ -#if defined(MBEDTLS_MD_C) -#define MBEDTLS_MD_LIGHT -#endif - -/* Auto-enable MBEDTLS_MD_LIGHT if needed by a module that didn't require it - * in a previous release, to ensure backwards compatibility. - */ -#if defined(MBEDTLS_ECJPAKE_C) || \ - defined(MBEDTLS_PEM_PARSE_C) || \ - defined(MBEDTLS_ENTROPY_C) || \ - defined(MBEDTLS_PK_C) || \ - defined(MBEDTLS_PKCS12_C) || \ - defined(MBEDTLS_RSA_C) || \ - defined(MBEDTLS_SSL_TLS_C) || \ - defined(MBEDTLS_X509_USE_C) || \ - defined(MBEDTLS_X509_CREATE_C) -#define MBEDTLS_MD_LIGHT -#endif - -#if defined(MBEDTLS_MD_LIGHT) -/* - * - MBEDTLS_MD_CAN_xxx is defined if the md module can perform xxx. - * - MBEDTLS_MD_xxx_VIA_PSA is defined if the md module may perform xxx via PSA - * (see below). - * - MBEDTLS_MD_SOME_PSA is defined if at least one algorithm may be performed - * via PSA (see below). - * - MBEDTLS_MD_SOME_LEGACY is defined if at least one algorithm may be performed - * via a direct legacy call (see below). - * - * The md module performs an algorithm via PSA if there is a PSA hash - * accelerator and the PSA driver subsytem is initialized at the time the - * operation is started, and makes a direct legacy call otherwise. - */ - -/* PSA accelerated implementations */ -#if defined(MBEDTLS_PSA_CRYPTO_C) - -#if defined(MBEDTLS_PSA_ACCEL_ALG_MD5) -#define MBEDTLS_MD_CAN_MD5 -#define MBEDTLS_MD_MD5_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA_1) -#define MBEDTLS_MD_CAN_SHA1 -#define MBEDTLS_MD_SHA1_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA_224) -#define MBEDTLS_MD_CAN_SHA224 -#define MBEDTLS_MD_SHA224_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA_256) -#define MBEDTLS_MD_CAN_SHA256 -#define MBEDTLS_MD_SHA256_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA_384) -#define MBEDTLS_MD_CAN_SHA384 -#define MBEDTLS_MD_SHA384_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA_512) -#define MBEDTLS_MD_CAN_SHA512 -#define MBEDTLS_MD_SHA512_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_RIPEMD160) -#define MBEDTLS_MD_CAN_RIPEMD160 -#define MBEDTLS_MD_RIPEMD160_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_224) -#define MBEDTLS_MD_CAN_SHA3_224 -#define MBEDTLS_MD_SHA3_224_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_256) -#define MBEDTLS_MD_CAN_SHA3_256 -#define MBEDTLS_MD_SHA3_256_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_384) -#define MBEDTLS_MD_CAN_SHA3_384 -#define MBEDTLS_MD_SHA3_384_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_512) -#define MBEDTLS_MD_CAN_SHA3_512 -#define MBEDTLS_MD_SHA3_512_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif - -#elif defined(MBEDTLS_PSA_CRYPTO_CLIENT) - -#if defined(PSA_WANT_ALG_MD5) -#define MBEDTLS_MD_CAN_MD5 -#define MBEDTLS_MD_MD5_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA_1) -#define MBEDTLS_MD_CAN_SHA1 -#define MBEDTLS_MD_SHA1_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA_224) -#define MBEDTLS_MD_CAN_SHA224 -#define MBEDTLS_MD_SHA224_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA_256) -#define MBEDTLS_MD_CAN_SHA256 -#define MBEDTLS_MD_SHA256_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA_384) -#define MBEDTLS_MD_CAN_SHA384 -#define MBEDTLS_MD_SHA384_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA_512) -#define MBEDTLS_MD_CAN_SHA512 -#define MBEDTLS_MD_SHA512_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_RIPEMD160) -#define MBEDTLS_MD_CAN_RIPEMD160 -#define MBEDTLS_MD_RIPEMD160_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA3_224) -#define MBEDTLS_MD_CAN_SHA3_224 -#define MBEDTLS_MD_SHA3_224_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA3_256) -#define MBEDTLS_MD_CAN_SHA3_256 -#define MBEDTLS_MD_SHA3_256_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA3_384) -#define MBEDTLS_MD_CAN_SHA3_384 -#define MBEDTLS_MD_SHA3_384_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif -#if defined(PSA_WANT_ALG_SHA3_512) -#define MBEDTLS_MD_CAN_SHA3_512 -#define MBEDTLS_MD_SHA3_512_VIA_PSA -#define MBEDTLS_MD_SOME_PSA -#endif - -#endif /* !MBEDTLS_PSA_CRYPTO_CLIENT && !MBEDTLS_PSA_CRYPTO_C */ - -/* Built-in implementations */ -#if defined(MBEDTLS_MD5_C) -#define MBEDTLS_MD_CAN_MD5 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA1_C) -#define MBEDTLS_MD_CAN_SHA1 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA224_C) -#define MBEDTLS_MD_CAN_SHA224 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA256_C) -#define MBEDTLS_MD_CAN_SHA256 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA384_C) -#define MBEDTLS_MD_CAN_SHA384 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA512_C) -#define MBEDTLS_MD_CAN_SHA512 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_SHA3_C) -#define MBEDTLS_MD_CAN_SHA3_224 -#define MBEDTLS_MD_CAN_SHA3_256 -#define MBEDTLS_MD_CAN_SHA3_384 -#define MBEDTLS_MD_CAN_SHA3_512 -#define MBEDTLS_MD_SOME_LEGACY -#endif -#if defined(MBEDTLS_RIPEMD160_C) -#define MBEDTLS_MD_CAN_RIPEMD160 -#define MBEDTLS_MD_SOME_LEGACY -#endif - -#endif /* MBEDTLS_MD_LIGHT */ - -/* BLOCK_CIPHER module can dispatch to PSA when: - * - PSA is enabled and drivers have been initialized - * - desired key type is supported on the PSA side - * If the above conditions are not met, but the legacy support is enabled, then - * BLOCK_CIPHER will dynamically fallback to it. - * - * In case BLOCK_CIPHER is defined (see below) the following symbols/helpers - * can be used to define its capabilities: - * - MBEDTLS_BLOCK_CIPHER_SOME_PSA: there is at least 1 key type between AES, - * ARIA and Camellia which is supported through a driver; - * - MBEDTLS_BLOCK_CIPHER_xxx_VIA_PSA: xxx key type is supported through a - * driver; - * - MBEDTLS_BLOCK_CIPHER_xxx_VIA_LEGACY: xxx key type is supported through - * a legacy module (i.e. MBEDTLS_xxx_C) - */ -#if defined(MBEDTLS_PSA_CRYPTO_C) -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_AES) -#define MBEDTLS_BLOCK_CIPHER_AES_VIA_PSA -#define MBEDTLS_BLOCK_CIPHER_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ARIA) -#define MBEDTLS_BLOCK_CIPHER_ARIA_VIA_PSA -#define MBEDTLS_BLOCK_CIPHER_SOME_PSA -#endif -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_CAMELLIA) -#define MBEDTLS_BLOCK_CIPHER_CAMELLIA_VIA_PSA -#define MBEDTLS_BLOCK_CIPHER_SOME_PSA -#endif -#endif /* MBEDTLS_PSA_CRYPTO_C */ - -#if defined(MBEDTLS_AES_C) -#define MBEDTLS_BLOCK_CIPHER_AES_VIA_LEGACY -#endif -#if defined(MBEDTLS_ARIA_C) -#define MBEDTLS_BLOCK_CIPHER_ARIA_VIA_LEGACY -#endif -#if defined(MBEDTLS_CAMELLIA_C) -#define MBEDTLS_BLOCK_CIPHER_CAMELLIA_VIA_LEGACY -#endif - -/* Helpers to state that BLOCK_CIPHER module supports AES, ARIA and/or Camellia - * block ciphers via either PSA or legacy. */ -#if defined(MBEDTLS_BLOCK_CIPHER_AES_VIA_PSA) || \ - defined(MBEDTLS_BLOCK_CIPHER_AES_VIA_LEGACY) -#define MBEDTLS_BLOCK_CIPHER_CAN_AES -#endif -#if defined(MBEDTLS_BLOCK_CIPHER_ARIA_VIA_PSA) || \ - defined(MBEDTLS_BLOCK_CIPHER_ARIA_VIA_LEGACY) -#define MBEDTLS_BLOCK_CIPHER_CAN_ARIA -#endif -#if defined(MBEDTLS_BLOCK_CIPHER_CAMELLIA_VIA_PSA) || \ - defined(MBEDTLS_BLOCK_CIPHER_CAMELLIA_VIA_LEGACY) -#define MBEDTLS_BLOCK_CIPHER_CAN_CAMELLIA -#endif - -/* GCM_C and CCM_C can either depend on (in order of preference) BLOCK_CIPHER_C - * or CIPHER_C. The former is auto-enabled when: - * - CIPHER_C is not defined, which is also the legacy solution; - * - BLOCK_CIPHER_SOME_PSA because in this case BLOCK_CIPHER can take advantage - * of the driver's acceleration. - */ -#if (defined(MBEDTLS_GCM_C) || defined(MBEDTLS_CCM_C)) && \ - (!defined(MBEDTLS_CIPHER_C) || defined(MBEDTLS_BLOCK_CIPHER_SOME_PSA)) -#define MBEDTLS_BLOCK_CIPHER_C -#endif - -/* Helpers for GCM/CCM capabilities */ -#if (defined(MBEDTLS_CIPHER_C) && defined(MBEDTLS_AES_C)) || \ - (defined(MBEDTLS_BLOCK_CIPHER_C) && defined(MBEDTLS_BLOCK_CIPHER_CAN_AES)) -#define MBEDTLS_CCM_GCM_CAN_AES -#endif - -#if (defined(MBEDTLS_CIPHER_C) && defined(MBEDTLS_ARIA_C)) || \ - (defined(MBEDTLS_BLOCK_CIPHER_C) && defined(MBEDTLS_BLOCK_CIPHER_CAN_ARIA)) -#define MBEDTLS_CCM_GCM_CAN_ARIA -#endif - -#if (defined(MBEDTLS_CIPHER_C) && defined(MBEDTLS_CAMELLIA_C)) || \ - (defined(MBEDTLS_BLOCK_CIPHER_C) && defined(MBEDTLS_BLOCK_CIPHER_CAN_CAMELLIA)) -#define MBEDTLS_CCM_GCM_CAN_CAMELLIA -#endif - -/* MBEDTLS_ECP_LIGHT is auto-enabled by the following symbols: - * - MBEDTLS_ECP_C because now it consists of MBEDTLS_ECP_LIGHT plus functions - * for curve arithmetic. As a consequence if MBEDTLS_ECP_C is required for - * some reason, then MBEDTLS_ECP_LIGHT should be enabled as well. - * - MBEDTLS_PK_PARSE_EC_EXTENDED and MBEDTLS_PK_PARSE_EC_COMPRESSED because - * these features are not supported in PSA so the only way to have them is - * to enable the built-in solution. - * Both of them are temporary dependencies: - * - PK_PARSE_EC_EXTENDED will be removed after #7779 and #7789 - * - support for compressed points should also be added to PSA, but in this - * case there is no associated issue to track it yet. - * - PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE because Weierstrass key derivation - * still depends on ECP_LIGHT. - * - PK_C + USE_PSA + PSA_WANT_ALG_ECDSA is a temporary dependency which will - * be fixed by #7453. - */ -#if defined(MBEDTLS_ECP_C) || \ - defined(MBEDTLS_PK_PARSE_EC_EXTENDED) || \ - defined(MBEDTLS_PK_PARSE_EC_COMPRESSED) || \ - defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#define MBEDTLS_ECP_LIGHT -#endif - -/* Backward compatibility: after #8740 the RSA module offers functions to parse - * and write RSA private/public keys without relying on the PK one. Of course - * this needs ASN1 support to do so, so we enable it here. */ -#if defined(MBEDTLS_RSA_C) -#define MBEDTLS_ASN1_PARSE_C -#define MBEDTLS_ASN1_WRITE_C -#endif - -/* MBEDTLS_PK_PARSE_EC_COMPRESSED is introduced in Mbed TLS version 3.5, while - * in previous version compressed points were automatically supported as long - * as PK_PARSE_C and ECP_C were enabled. As a consequence, for backward - * compatibility, we auto-enable PK_PARSE_EC_COMPRESSED when these conditions - * are met. */ -#if defined(MBEDTLS_PK_PARSE_C) && defined(MBEDTLS_ECP_C) -#define MBEDTLS_PK_PARSE_EC_COMPRESSED -#endif - -/* Helper symbol to state that there is support for ECDH, either through - * library implementation (ECDH_C) or through PSA. */ -#if (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_ALG_ECDH)) || \ - (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_ECDH_C)) -#define MBEDTLS_CAN_ECDH -#endif - -/* PK module can achieve ECDSA functionalities by means of either software - * implementations (ECDSA_C) or through a PSA driver. The following defines - * are meant to list these capabilities in a general way which abstracts how - * they are implemented under the hood. */ -#if !defined(MBEDTLS_USE_PSA_CRYPTO) -#if defined(MBEDTLS_ECDSA_C) -#define MBEDTLS_PK_CAN_ECDSA_SIGN -#define MBEDTLS_PK_CAN_ECDSA_VERIFY -#endif /* MBEDTLS_ECDSA_C */ -#else /* MBEDTLS_USE_PSA_CRYPTO */ -#if defined(PSA_WANT_ALG_ECDSA) -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#define MBEDTLS_PK_CAN_ECDSA_SIGN -#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC */ -#if defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) -#define MBEDTLS_PK_CAN_ECDSA_VERIFY -#endif /* PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY */ -#endif /* PSA_WANT_ALG_ECDSA */ -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -#if defined(MBEDTLS_PK_CAN_ECDSA_VERIFY) || defined(MBEDTLS_PK_CAN_ECDSA_SIGN) -#define MBEDTLS_PK_CAN_ECDSA_SOME -#endif - -/* Helpers to state that each key is supported either on the builtin or PSA side. */ -#if defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) || defined(PSA_WANT_ECC_SECP_R1_521) -#define MBEDTLS_ECP_HAVE_SECP521R1 -#endif -#if defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) || defined(PSA_WANT_ECC_BRAINPOOL_P_R1_512) -#define MBEDTLS_ECP_HAVE_BP512R1 -#endif -#if defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) || defined(PSA_WANT_ECC_MONTGOMERY_448) -#define MBEDTLS_ECP_HAVE_CURVE448 -#endif -#if defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) || defined(PSA_WANT_ECC_BRAINPOOL_P_R1_384) -#define MBEDTLS_ECP_HAVE_BP384R1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) || defined(PSA_WANT_ECC_SECP_R1_384) -#define MBEDTLS_ECP_HAVE_SECP384R1 -#endif -#if defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) || defined(PSA_WANT_ECC_BRAINPOOL_P_R1_256) -#define MBEDTLS_ECP_HAVE_BP256R1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) || defined(PSA_WANT_ECC_SECP_K1_256) -#define MBEDTLS_ECP_HAVE_SECP256K1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) || defined(PSA_WANT_ECC_SECP_R1_256) -#define MBEDTLS_ECP_HAVE_SECP256R1 -#endif -#if defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) || defined(PSA_WANT_ECC_MONTGOMERY_255) -#define MBEDTLS_ECP_HAVE_CURVE25519 -#endif -#if defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) || defined(PSA_WANT_ECC_SECP_K1_224) -#define MBEDTLS_ECP_HAVE_SECP224K1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) || defined(PSA_WANT_ECC_SECP_R1_224) -#define MBEDTLS_ECP_HAVE_SECP224R1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) || defined(PSA_WANT_ECC_SECP_K1_192) -#define MBEDTLS_ECP_HAVE_SECP192K1 -#endif -#if defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) || defined(PSA_WANT_ECC_SECP_R1_192) -#define MBEDTLS_ECP_HAVE_SECP192R1 -#endif - -/* Helper symbol to state that the PK module has support for EC keys. This - * can either be provided through the legacy ECP solution or through the - * PSA friendly MBEDTLS_PK_USE_PSA_EC_DATA (see pk.h for its description). */ -#if defined(MBEDTLS_ECP_C) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY)) -#define MBEDTLS_PK_HAVE_ECC_KEYS -#endif /* MBEDTLS_PK_USE_PSA_EC_DATA || MBEDTLS_ECP_C */ - -/* Historically pkparse did not check the CBC padding when decrypting - * a key. This was a bug, which is now fixed. As a consequence, pkparse - * now needs PKCS7 padding support, but existing configurations might not - * enable it, so we enable it here. */ -#if defined(MBEDTLS_PK_PARSE_C) && defined(MBEDTLS_PKCS5_C) && defined(MBEDTLS_CIPHER_MODE_CBC) -#define MBEDTLS_CIPHER_PADDING_PKCS7 -#endif - -/* Backwards compatibility for some macros which were renamed to reflect that - * they are related to Armv8, not aarch64. */ -#if defined(MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT) && \ - !defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT) -#define MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT -#endif -#if defined(MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY) && !defined(MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY) -#define MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY -#endif - -/* psa_util file features some ECDSA conversion functions, to convert between - * legacy's ASN.1 DER format and PSA's raw one. */ -#if (defined(MBEDTLS_PSA_CRYPTO_CLIENT) && \ - (defined(PSA_WANT_ALG_ECDSA) || defined(PSA_WANT_ALG_DETERMINISTIC_ECDSA))) -#define MBEDTLS_PSA_UTIL_HAVE_ECDSA -#endif - -/* Some internal helpers to determine which keys are available. */ -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_AES_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_KEY_TYPE_AES)) -#define MBEDTLS_SSL_HAVE_AES -#endif -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_ARIA_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_KEY_TYPE_ARIA)) -#define MBEDTLS_SSL_HAVE_ARIA -#endif -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_CAMELLIA_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_KEY_TYPE_CAMELLIA)) -#define MBEDTLS_SSL_HAVE_CAMELLIA -#endif - -/* Some internal helpers to determine which operation modes are available. */ -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_CIPHER_MODE_CBC)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_ALG_CBC_NO_PADDING)) -#define MBEDTLS_SSL_HAVE_CBC -#endif - -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_GCM_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_ALG_GCM)) -#define MBEDTLS_SSL_HAVE_GCM -#endif - -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_CCM_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_ALG_CCM)) -#define MBEDTLS_SSL_HAVE_CCM -#endif - -#if (!defined(MBEDTLS_USE_PSA_CRYPTO) && defined(MBEDTLS_CHACHAPOLY_C)) || \ - (defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_ALG_CHACHA20_POLY1305)) -#define MBEDTLS_SSL_HAVE_CHACHAPOLY -#endif - -#if defined(MBEDTLS_SSL_HAVE_GCM) || defined(MBEDTLS_SSL_HAVE_CCM) || \ - defined(MBEDTLS_SSL_HAVE_CHACHAPOLY) -#define MBEDTLS_SSL_HAVE_AEAD -#endif - -#endif /* MBEDTLS_CONFIG_ADJUST_LEGACY_CRYPTO_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_from_psa.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_from_psa.h deleted file mode 100644 index 48f1bab1e..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_legacy_from_psa.h +++ /dev/null @@ -1,873 +0,0 @@ -/** - * \file mbedtls/config_adjust_legacy_from_psa.h - * \brief Adjust PSA configuration: activate legacy implementations - * - * This is an internal header. Do not include it directly. - * - * When MBEDTLS_PSA_CRYPTO_CONFIG is enabled, activate legacy implementations - * of cryptographic mechanisms as needed to fulfill the needs of the PSA - * configuration. Generally speaking, we activate a legacy mechanism if - * it's needed for a requested PSA mechanism and there is no PSA driver - * for it. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_LEGACY_FROM_PSA_H -#define MBEDTLS_CONFIG_ADJUST_LEGACY_FROM_PSA_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/* Define appropriate ACCEL macros for the p256-m driver. - * In the future, those should be generated from the drivers JSON description. - */ -#if defined(MBEDTLS_PSA_P256M_DRIVER_ENABLED) -#define MBEDTLS_PSA_ACCEL_ECC_SECP_R1_256 -#define MBEDTLS_PSA_ACCEL_ALG_ECDSA -#define MBEDTLS_PSA_ACCEL_ALG_ECDH -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_GENERATE -#endif - -/* - * ECC: support for a feature is controlled by a triplet or a pair: - * (curve, key_type public/basic, alg) or (curve, key_type_). - * - * A triplet/pair is accelerated if all of is components are accelerated; - * otherwise each component needs to be built in. - * - * We proceed in two passes: - * 1. Check if acceleration is complete for curves, key types, algs. - * 2. Then enable built-ins for each thing that's either not accelerated of - * doesn't have complete acceleration of the other triplet/pair components. - * - * Note: this needs psa/crypto_adjust_keypair_types.h to have been included - * already, so that we know the full set of key types that are requested. - */ - -/* ECC: curves: is acceleration complete? */ -#if (defined(PSA_WANT_ECC_BRAINPOOL_P_R1_256) && \ - !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_256)) || \ - (defined(PSA_WANT_ECC_BRAINPOOL_P_R1_384) && \ - !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_384)) || \ - (defined(PSA_WANT_ECC_BRAINPOOL_P_R1_512) && \ - !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_512)) || \ - (defined(PSA_WANT_ECC_SECP_R1_192) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_192)) || \ - (defined(PSA_WANT_ECC_SECP_R1_224) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_224)) || \ - (defined(PSA_WANT_ECC_SECP_R1_256) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_256)) || \ - (defined(PSA_WANT_ECC_SECP_R1_384) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_384)) || \ - (defined(PSA_WANT_ECC_SECP_R1_521) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_521)) || \ - (defined(PSA_WANT_ECC_SECP_K1_192) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_192)) || \ - (defined(PSA_WANT_ECC_SECP_K1_256) && !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_256)) -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_WEIERSTRASS_CURVES -#endif - -#if (defined(PSA_WANT_ECC_MONTGOMERY_255) && !defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_255)) || \ - (defined(PSA_WANT_ECC_MONTGOMERY_448) && !defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_448)) -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES -#endif - -/* ECC: algs: is acceleration complete? */ -#if (defined(PSA_WANT_ALG_ECDH) && !defined(MBEDTLS_PSA_ACCEL_ALG_ECDH)) || \ - (defined(PSA_WANT_ALG_ECDSA) && !defined(MBEDTLS_PSA_ACCEL_ALG_ECDSA)) || \ - (defined(PSA_WANT_ALG_DETERMINISTIC_ECDSA) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_DETERMINISTIC_ECDSA)) || \ - (defined(PSA_WANT_ALG_JPAKE) && !defined(MBEDTLS_PSA_ACCEL_ALG_JPAKE)) -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS -#endif - -/* ECC: key types: is acceleration complete? */ -#if (defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC)) -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES_BASIC -#endif - -/* Special case: we don't support cooked key derivation in drivers yet */ -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#undef MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE -#endif - -/* Note: the condition about key derivation is always true as DERIVE can't be - * accelerated yet */ -#if (defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_GENERATE)) || \ - (defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE)) -#define MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES -#endif - -/* ECC: curves: enable built-ins as needed. - * - * We need the curve built-in: - * - if it's not accelerated, or - * - if there's a key type with missing acceleration, or - * - if there's a alg with missing acceleration. - */ -#if defined(PSA_WANT_ECC_BRAINPOOL_P_R1_256) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_256) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_256 1 -#define MBEDTLS_ECP_DP_BP256R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_256 */ - -#if defined(PSA_WANT_ECC_BRAINPOOL_P_R1_384) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_384) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_384 1 -#define MBEDTLS_ECP_DP_BP384R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_384 */ - -#if defined(PSA_WANT_ECC_BRAINPOOL_P_R1_512) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_BRAINPOOL_P_R1_512) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_512 1 -#define MBEDTLS_ECP_DP_BP512R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_512 */ - -#if defined(PSA_WANT_ECC_MONTGOMERY_255) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_255) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_MONTGOMERY_255 1 -#define MBEDTLS_ECP_DP_CURVE25519_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_MONTGOMERY_255 */ - -#if defined(PSA_WANT_ECC_MONTGOMERY_448) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_MONTGOMERY_448) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_MONTGOMERY_448 1 -#define MBEDTLS_ECP_DP_CURVE448_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_MONTGOMERY_448 */ - -#if defined(PSA_WANT_ECC_SECP_R1_192) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_192) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_192 1 -#define MBEDTLS_ECP_DP_SECP192R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_R1_192 */ - -#if defined(PSA_WANT_ECC_SECP_R1_224) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_224) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_224 1 -#define MBEDTLS_ECP_DP_SECP224R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_R1_224 */ - -#if defined(PSA_WANT_ECC_SECP_R1_256) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_256) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_256 1 -#define MBEDTLS_ECP_DP_SECP256R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_R1_256 */ - -#if defined(PSA_WANT_ECC_SECP_R1_384) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_384) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_384 1 -#define MBEDTLS_ECP_DP_SECP384R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_R1_384 */ - -#if defined(PSA_WANT_ECC_SECP_R1_521) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_521) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_521 1 -#define MBEDTLS_ECP_DP_SECP521R1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_R1_521 */ - -#if defined(PSA_WANT_ECC_SECP_K1_192) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_192) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_K1_192 1 -#define MBEDTLS_ECP_DP_SECP192K1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_K1_192 */ - -#if defined(PSA_WANT_ECC_SECP_K1_256) -#if !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_K1_256) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_K1_256 1 -#define MBEDTLS_ECP_DP_SECP256K1_ENABLED -#endif /* missing accel */ -#endif /* PSA_WANT_ECC_SECP_K1_256 */ - -/* ECC: algs: enable built-ins as needed. - * - * We need the alg built-in: - * - if it's not accelerated, or - * - if there's a relevant curve (see below) with missing acceleration, or - * - if there's a key type among (public, basic) with missing acceleration. - * - * Relevant curves are: - * - all curves for ECDH - * - Weierstrass curves for (deterministic) ECDSA - * - secp256r1 for EC J-PAKE - */ -#if defined(PSA_WANT_ALG_DETERMINISTIC_ECDSA) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_DETERMINISTIC_ECDSA) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_WEIERSTRASS_CURVES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES_BASIC) -#define MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA 1 -#define MBEDTLS_ECDSA_DETERMINISTIC -#define MBEDTLS_HMAC_DRBG_C -#define MBEDTLS_MD_C -#define MBEDTLS_ECDSA_C -#define MBEDTLS_ECP_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_ASN1_PARSE_C -#define MBEDTLS_ASN1_WRITE_C -#endif /* missing accel */ -#endif /* PSA_WANT_ALG_DETERMINISTIC_ECDSA */ - -#if defined(PSA_WANT_ALG_ECDH) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_ECDH) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES_BASIC) -#define MBEDTLS_PSA_BUILTIN_ALG_ECDH 1 -#define MBEDTLS_ECDH_C -#define MBEDTLS_ECP_C -#define MBEDTLS_BIGNUM_C -#endif /* missing accel */ -#endif /* PSA_WANT_ALG_ECDH */ - -#if defined(PSA_WANT_ALG_ECDSA) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_ECDSA) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_WEIERSTRASS_CURVES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES_BASIC) -#define MBEDTLS_PSA_BUILTIN_ALG_ECDSA 1 -#define MBEDTLS_ECDSA_C -#define MBEDTLS_ECP_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_ASN1_PARSE_C -#define MBEDTLS_ASN1_WRITE_C -#endif /* missing accel */ -#endif /* PSA_WANT_ALG_ECDSA */ - -#if defined(PSA_WANT_ALG_JPAKE) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_JPAKE) || \ - !defined(MBEDTLS_PSA_ACCEL_ECC_SECP_R1_256) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_KEY_TYPES_BASIC) -#define MBEDTLS_PSA_BUILTIN_PAKE 1 -#define MBEDTLS_PSA_BUILTIN_ALG_JPAKE 1 -#define MBEDTLS_ECP_DP_SECP256R1_ENABLED -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_ECP_C -#define MBEDTLS_ECJPAKE_C -#endif /* missing accel */ -#endif /* PSA_WANT_ALG_JPAKE */ - -/* ECC: key types: enable built-ins as needed. - * - * We need the key type built-in: - * - if it's not accelerated, or - * - if there's a curve with missing acceleration, or - * - only for public/basic: if there's an alg with missing acceleration. - */ -#if defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_PUBLIC_KEY) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_PUBLIC_KEY 1 -#endif /* missing accel */ -#endif /* PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY */ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#endif /* missing accel */ -#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC */ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#endif /* missing accel */ -#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT */ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#endif /* missing accel */ -#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT */ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_GENERATE) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_GENERATE 1 -#endif /* missing accel */ -#endif /* PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE */ - -/* Note: the condition is always true as DERIVE can't be accelerated yet */ -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE) || \ - defined(MBEDTLS_PSA_ECC_ACCEL_INCOMPLETE_CURVES) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_DERIVE 1 -#endif /* missing accel */ -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE */ - -#if defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_PUBLIC_KEY) || \ - defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_BASIC) || \ - defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ - defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#define MBEDTLS_ECP_LIGHT -#define MBEDTLS_BIGNUM_C -#endif - -#if defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_EXPORT) || \ - defined(MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_GENERATE) -#define MBEDTLS_ECP_C -#define MBEDTLS_BIGNUM_C -#endif - -/* End of ECC section */ - -/* - * DH key types follow the same pattern used above for EC keys. They are defined - * by a triplet (group, key_type, alg). A triplet is accelerated if all its - * component are accelerated, otherwise each component needs to be builtin. - */ - -/* DH: groups: is acceleration complete? */ -#if (defined(PSA_WANT_DH_RFC7919_2048) && !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_2048)) || \ - (defined(PSA_WANT_DH_RFC7919_3072) && !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_3072)) || \ - (defined(PSA_WANT_DH_RFC7919_4096) && !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_4096)) || \ - (defined(PSA_WANT_DH_RFC7919_6144) && !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_6144)) || \ - (defined(PSA_WANT_DH_RFC7919_8192) && !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_8192)) -#define MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS -#endif - -/* DH: algs: is acceleration complete? */ -#if defined(PSA_WANT_ALG_FFDH) && !defined(MBEDTLS_PSA_ACCEL_ALG_FFDH) -#define MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS -#endif - -/* DH: key types: is acceleration complete? */ -#if (defined(PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_PUBLIC_KEY)) || \ - (defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_BASIC)) || \ - (defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_IMPORT)) || \ - (defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_EXPORT)) || \ - (defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE) && \ - !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_GENERATE)) -#define MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES -#endif - -#if defined(PSA_WANT_DH_RFC7919_2048) -#if !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_2048) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_2048 1 -#endif /* !MBEDTLS_PSA_BUILTIN_DH_RFC7919_2048 */ -#endif /* PSA_WANT_DH_RFC7919_2048 */ - -#if defined(PSA_WANT_DH_RFC7919_3072) -#if !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_3072) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_3072 1 -#endif /* !MBEDTLS_PSA_BUILTIN_DH_RFC7919_3072 */ -#endif /* PSA_WANT_DH_RFC7919_3072 */ - -#if defined(PSA_WANT_DH_RFC7919_4096) -#if !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_4096) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_4096 1 -#endif /* !MBEDTLS_PSA_BUILTIN_DH_RFC7919_4096 */ -#endif /* PSA_WANT_DH_RFC7919_4096 */ - -#if defined(PSA_WANT_DH_RFC7919_6144) -#if !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_6144) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_6144 1 -#endif /* !MBEDTLS_PSA_BUILTIN_DH_RFC7919_6144 */ -#endif /* PSA_WANT_DH_RFC7919_6144 */ - -#if defined(PSA_WANT_DH_RFC7919_8192) -#if !defined(MBEDTLS_PSA_ACCEL_DH_RFC7919_8192) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_8192 1 -#endif /* !MBEDTLS_PSA_BUILTIN_DH_RFC7919_8192 */ -#endif /* PSA_WANT_DH_RFC7919_8192 */ - -#if defined(PSA_WANT_ALG_FFDH) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_FFDH) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_KEY_TYPES) -#define MBEDTLS_PSA_BUILTIN_ALG_FFDH 1 -#define MBEDTLS_BIGNUM_C -#endif /* !MBEDTLS_PSA_ACCEL_ALG_FFDH */ -#endif /* PSA_WANT_ALG_FFDH */ - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_IMPORT) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_IMPORT 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_IMPORT */ -#endif /* PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT */ - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_EXPORT) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_EXPORT 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_EXPORT */ -#endif /* PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT */ - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_GENERATE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_GENERATE 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_GENERATE */ -#endif /* PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE */ - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_BASIC) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_BASIC 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_KEY_PAIR_BASIC */ -#endif /* PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC */ - -#if defined(PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_PUBLIC_KEY) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_GROUPS) || \ - defined(MBEDTLS_PSA_DH_ACCEL_INCOMPLETE_ALGS) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_PUBLIC_KEY 1 -#define MBEDTLS_BIGNUM_C -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DH_PUBLIC_KEY */ -#endif /* PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY */ - -/* End of DH section */ - -#if defined(PSA_WANT_ALG_HKDF) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF) -/* - * The PSA implementation has its own implementation of HKDF, separate from - * hkdf.c. No need to enable MBEDTLS_HKDF_C here. - */ -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_HKDF */ -#endif /* PSA_WANT_ALG_HKDF */ - -#if defined(PSA_WANT_ALG_HKDF_EXTRACT) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF_EXTRACT) -/* - * The PSA implementation has its own implementation of HKDF, separate from - * hkdf.c. No need to enable MBEDTLS_HKDF_C here. - */ -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXTRACT 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_HKDF_EXTRACT */ -#endif /* PSA_WANT_ALG_HKDF_EXTRACT */ - -#if defined(PSA_WANT_ALG_HKDF_EXPAND) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF_EXPAND) -/* - * The PSA implementation has its own implementation of HKDF, separate from - * hkdf.c. No need to enable MBEDTLS_HKDF_C here. - */ -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXPAND 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_HKDF_EXPAND */ -#endif /* PSA_WANT_ALG_HKDF_EXPAND */ - -#if defined(PSA_WANT_ALG_HMAC) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_HMAC) -#define MBEDTLS_PSA_BUILTIN_ALG_HMAC 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_HMAC */ -#endif /* PSA_WANT_ALG_HMAC */ - -#if defined(PSA_WANT_ALG_MD5) && !defined(MBEDTLS_PSA_ACCEL_ALG_MD5) -#define MBEDTLS_PSA_BUILTIN_ALG_MD5 1 -#define MBEDTLS_MD5_C -#endif - -#if defined(PSA_WANT_ALG_RIPEMD160) && !defined(MBEDTLS_PSA_ACCEL_ALG_RIPEMD160) -#define MBEDTLS_PSA_BUILTIN_ALG_RIPEMD160 1 -#define MBEDTLS_RIPEMD160_C -#endif - -#if defined(PSA_WANT_ALG_RSA_OAEP) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_RSA_OAEP) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_OAEP 1 -#define MBEDTLS_RSA_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_OID_C -#define MBEDTLS_PKCS1_V21 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_RSA_OAEP */ -#endif /* PSA_WANT_ALG_RSA_OAEP */ - -#if defined(PSA_WANT_ALG_RSA_PKCS1V15_CRYPT) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_RSA_PKCS1V15_CRYPT) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PKCS1V15_CRYPT 1 -#define MBEDTLS_RSA_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_OID_C -#define MBEDTLS_PKCS1_V15 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_RSA_PKCS1V15_CRYPT */ -#endif /* PSA_WANT_ALG_RSA_PKCS1V15_CRYPT */ - -#if defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_RSA_PKCS1V15_SIGN) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PKCS1V15_SIGN 1 -#define MBEDTLS_RSA_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_OID_C -#define MBEDTLS_PKCS1_V15 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_RSA_PKCS1V15_SIGN */ -#endif /* PSA_WANT_ALG_RSA_PKCS1V15_SIGN */ - -#if defined(PSA_WANT_ALG_RSA_PSS) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_RSA_PSS) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PSS 1 -#define MBEDTLS_RSA_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_OID_C -#define MBEDTLS_PKCS1_V21 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_RSA_PSS */ -#endif /* PSA_WANT_ALG_RSA_PSS */ - -#if defined(PSA_WANT_ALG_SHA_1) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA_1) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_1 1 -#define MBEDTLS_SHA1_C -#endif - -#if defined(PSA_WANT_ALG_SHA_224) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA_224) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_224 1 -#define MBEDTLS_SHA224_C -#endif - -#if defined(PSA_WANT_ALG_SHA_256) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA_256) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_256 1 -#define MBEDTLS_SHA256_C -#endif - -#if defined(PSA_WANT_ALG_SHA_384) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA_384) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_384 1 -#define MBEDTLS_SHA384_C -#endif - -#if defined(PSA_WANT_ALG_SHA_512) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA_512) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_512 1 -#define MBEDTLS_SHA512_C -#endif - -#if defined(PSA_WANT_ALG_SHA3_224) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_224) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_224 1 -#define MBEDTLS_SHA3_C -#endif - -#if defined(PSA_WANT_ALG_SHA3_256) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_256) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_256 1 -#define MBEDTLS_SHA3_C -#endif - -#if defined(PSA_WANT_ALG_SHA3_384) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_384) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_384 1 -#define MBEDTLS_SHA3_C -#endif - -#if defined(PSA_WANT_ALG_SHA3_512) && !defined(MBEDTLS_PSA_ACCEL_ALG_SHA3_512) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_512 1 -#define MBEDTLS_SHA3_C -#endif - -#if defined(PSA_WANT_ALG_PBKDF2_HMAC) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_PBKDF2_HMAC) -#define MBEDTLS_PSA_BUILTIN_ALG_PBKDF2_HMAC 1 -#define PSA_HAVE_SOFT_PBKDF2_HMAC 1 -#endif /* !MBEDTLS_PSA_BUILTIN_ALG_PBKDF2_HMAC */ -#endif /* PSA_WANT_ALG_PBKDF2_HMAC */ - -#if defined(PSA_WANT_ALG_TLS12_PRF) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_TLS12_PRF) -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_PRF 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_TLS12_PRF */ -#endif /* PSA_WANT_ALG_TLS12_PRF */ - -#if defined(PSA_WANT_ALG_TLS12_PSK_TO_MS) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_TLS12_PSK_TO_MS) -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_TLS12_PSK_TO_MS */ -#endif /* PSA_WANT_ALG_TLS12_PSK_TO_MS */ - -#if defined(PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_TLS12_ECJPAKE_TO_PMS) -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_ECJPAKE_TO_PMS 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_TLS12_ECJPAKE_TO_PMS */ -#endif /* PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS */ - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_IMPORT) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_IMPORT */ -#endif /* PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT */ - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_EXPORT) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_EXPORT */ -#endif /* PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT */ - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_GENERATE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_GENERATE 1 -#define MBEDTLS_GENPRIME -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_GENERATE */ -#endif /* PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE */ - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_BASIC */ -#endif /* PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC */ - -#if defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_PUBLIC_KEY) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_PUBLIC_KEY 1 -#define MBEDTLS_RSA_C -#define MBEDTLS_BIGNUM_C -#define MBEDTLS_OID_C -#define MBEDTLS_ASN1_PARSE_C -#define MBEDTLS_ASN1_WRITE_C -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_PUBLIC_KEY */ -#endif /* PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY */ - -/* If any of the block modes are requested that don't have an - * associated HW assist, define PSA_HAVE_SOFT_BLOCK_MODE for checking - * in the block cipher key types. */ -#if (defined(PSA_WANT_ALG_CTR) && !defined(MBEDTLS_PSA_ACCEL_ALG_CTR)) || \ - (defined(PSA_WANT_ALG_CFB) && !defined(MBEDTLS_PSA_ACCEL_ALG_CFB)) || \ - (defined(PSA_WANT_ALG_OFB) && !defined(MBEDTLS_PSA_ACCEL_ALG_OFB)) || \ - (defined(PSA_WANT_ALG_ECB_NO_PADDING) && !defined(MBEDTLS_PSA_ACCEL_ALG_ECB_NO_PADDING)) || \ - (defined(PSA_WANT_ALG_CBC_NO_PADDING) && !defined(MBEDTLS_PSA_ACCEL_ALG_CBC_NO_PADDING)) || \ - (defined(PSA_WANT_ALG_CBC_PKCS7) && !defined(MBEDTLS_PSA_ACCEL_ALG_CBC_PKCS7)) || \ - (defined(PSA_WANT_ALG_CMAC) && !defined(MBEDTLS_PSA_ACCEL_ALG_CMAC)) -#define PSA_HAVE_SOFT_BLOCK_MODE 1 -#endif - -#if defined(PSA_WANT_ALG_PBKDF2_AES_CMAC_PRF_128) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_PBKDF2_AES_CMAC_PRF_128) -#define MBEDTLS_PSA_BUILTIN_ALG_PBKDF2_AES_CMAC_PRF_128 1 -#define PSA_HAVE_SOFT_PBKDF2_CMAC 1 -#endif /* !MBEDTLS_PSA_ACCEL_ALG_PBKDF2_AES_CMAC_PRF_128 */ -#endif /* PSA_WANT_ALG_PBKDF2_AES_CMAC_PRF_128 */ - -#if defined(PSA_WANT_KEY_TYPE_AES) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_AES) -#define PSA_HAVE_SOFT_KEY_TYPE_AES 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_AES */ -#if defined(PSA_HAVE_SOFT_KEY_TYPE_AES) || \ - defined(PSA_HAVE_SOFT_BLOCK_MODE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_AES 1 -#define MBEDTLS_AES_C -#endif /* PSA_HAVE_SOFT_KEY_TYPE_AES || PSA_HAVE_SOFT_BLOCK_MODE */ -#endif /* PSA_WANT_KEY_TYPE_AES */ - -#if defined(PSA_WANT_KEY_TYPE_ARIA) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ARIA) -#define PSA_HAVE_SOFT_KEY_TYPE_ARIA 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_ARIA */ -#if defined(PSA_HAVE_SOFT_KEY_TYPE_ARIA) || \ - defined(PSA_HAVE_SOFT_BLOCK_MODE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ARIA 1 -#define MBEDTLS_ARIA_C -#endif /* PSA_HAVE_SOFT_KEY_TYPE_ARIA || PSA_HAVE_SOFT_BLOCK_MODE */ -#endif /* PSA_WANT_KEY_TYPE_ARIA */ - -#if defined(PSA_WANT_KEY_TYPE_CAMELLIA) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_CAMELLIA) -#define PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_CAMELLIA */ -#if defined(PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA) || \ - defined(PSA_HAVE_SOFT_BLOCK_MODE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_CAMELLIA 1 -#define MBEDTLS_CAMELLIA_C -#endif /* PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA || PSA_HAVE_SOFT_BLOCK_MODE */ -#endif /* PSA_WANT_KEY_TYPE_CAMELLIA */ - -#if defined(PSA_WANT_KEY_TYPE_DES) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_DES) -#define PSA_HAVE_SOFT_KEY_TYPE_DES 1 -#endif /* !MBEDTLS_PSA_ACCEL_KEY_TYPE_DES */ -#if defined(PSA_HAVE_SOFT_KEY_TYPE_DES) || \ - defined(PSA_HAVE_SOFT_BLOCK_MODE) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DES 1 -#define MBEDTLS_DES_C -#endif /*PSA_HAVE_SOFT_KEY_TYPE_DES || PSA_HAVE_SOFT_BLOCK_MODE */ -#endif /* PSA_WANT_KEY_TYPE_DES */ - -#if defined(PSA_WANT_ALG_STREAM_CIPHER) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_STREAM_CIPHER) -#define MBEDTLS_PSA_BUILTIN_ALG_STREAM_CIPHER 1 -#endif /* MBEDTLS_PSA_ACCEL_ALG_STREAM_CIPHER */ -#endif /* PSA_WANT_ALG_STREAM_CIPHER */ - -#if defined(PSA_WANT_KEY_TYPE_CHACHA20) -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_CHACHA20) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_STREAM_CIPHER) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_CHACHA20 1 -#define MBEDTLS_CHACHA20_C -#endif /*!MBEDTLS_PSA_ACCEL_KEY_TYPE_CHACHA20 */ -#endif /* PSA_WANT_KEY_TYPE_CHACHA20 */ - -/* If any of the software block ciphers are selected, define - * PSA_HAVE_SOFT_BLOCK_CIPHER, which can be used in any of these - * situations. */ -#if defined(PSA_HAVE_SOFT_KEY_TYPE_AES) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_ARIA) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_DES) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA) -#define PSA_HAVE_SOFT_BLOCK_CIPHER 1 -#endif - -#if defined(PSA_WANT_ALG_CMAC) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CMAC) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_PSA_BUILTIN_ALG_CMAC 1 -#define MBEDTLS_CMAC_C -#endif /* !MBEDTLS_PSA_ACCEL_ALG_CMAC */ -#endif /* PSA_WANT_ALG_CMAC */ - -#if defined(PSA_HAVE_SOFT_PBKDF2_HMAC) || \ - defined(PSA_HAVE_SOFT_PBKDF2_CMAC) -#define PSA_HAVE_SOFT_PBKDF2 1 -#endif /* PSA_HAVE_SOFT_PBKDF2_HMAC || PSA_HAVE_SOFT_PBKDF2_CMAC */ - -#if defined(PSA_WANT_ALG_CTR) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CTR) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_PSA_BUILTIN_ALG_CTR 1 -#define MBEDTLS_CIPHER_MODE_CTR -#endif -#endif /* PSA_WANT_ALG_CTR */ - -#if defined(PSA_WANT_ALG_CFB) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CFB) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_PSA_BUILTIN_ALG_CFB 1 -#define MBEDTLS_CIPHER_MODE_CFB -#endif -#endif /* PSA_WANT_ALG_CFB */ - -#if defined(PSA_WANT_ALG_OFB) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_OFB) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_PSA_BUILTIN_ALG_OFB 1 -#define MBEDTLS_CIPHER_MODE_OFB -#endif -#endif /* PSA_WANT_ALG_OFB */ - -#if defined(PSA_WANT_ALG_ECB_NO_PADDING) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_ECB_NO_PADDING) -#define MBEDTLS_PSA_BUILTIN_ALG_ECB_NO_PADDING 1 -#endif - -#if defined(PSA_WANT_ALG_CBC_NO_PADDING) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CBC_NO_PADDING) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_CIPHER_MODE_CBC -#define MBEDTLS_PSA_BUILTIN_ALG_CBC_NO_PADDING 1 -#endif -#endif /* PSA_WANT_ALG_CBC_NO_PADDING */ - -#if defined(PSA_WANT_ALG_CBC_PKCS7) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CBC_PKCS7) || \ - defined(PSA_HAVE_SOFT_BLOCK_CIPHER) -#define MBEDTLS_CIPHER_MODE_CBC -#define MBEDTLS_PSA_BUILTIN_ALG_CBC_PKCS7 1 -#define MBEDTLS_CIPHER_PADDING_PKCS7 -#endif -#endif /* PSA_WANT_ALG_CBC_PKCS7 */ - -#if defined(PSA_WANT_ALG_CCM) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CCM) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_AES) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_ARIA) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA) -#define MBEDTLS_PSA_BUILTIN_ALG_CCM 1 -#define MBEDTLS_CCM_C -#endif -#endif /* PSA_WANT_ALG_CCM */ - -#if defined(PSA_WANT_ALG_CCM_STAR_NO_TAG) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CCM_STAR_NO_TAG) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_AES) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_ARIA) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA) -#define MBEDTLS_PSA_BUILTIN_ALG_CCM_STAR_NO_TAG 1 -#define MBEDTLS_CCM_C -#endif -#endif /* PSA_WANT_ALG_CCM_STAR_NO_TAG */ - -#if defined(PSA_WANT_ALG_GCM) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_GCM) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_AES) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_ARIA) || \ - defined(PSA_HAVE_SOFT_KEY_TYPE_CAMELLIA) -#define MBEDTLS_PSA_BUILTIN_ALG_GCM 1 -#define MBEDTLS_GCM_C -#endif -#endif /* PSA_WANT_ALG_GCM */ - -#if defined(PSA_WANT_ALG_CHACHA20_POLY1305) -#if !defined(MBEDTLS_PSA_ACCEL_ALG_CHACHA20_POLY1305) -#if defined(PSA_WANT_KEY_TYPE_CHACHA20) -#define MBEDTLS_CHACHAPOLY_C -#define MBEDTLS_CHACHA20_C -#define MBEDTLS_POLY1305_C -#define MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305 1 -#endif /* PSA_WANT_KEY_TYPE_CHACHA20 */ -#endif /* !MBEDTLS_PSA_ACCEL_ALG_CHACHA20_POLY1305 */ -#endif /* PSA_WANT_ALG_CHACHA20_POLY1305 */ - -#endif /* MBEDTLS_CONFIG_ADJUST_LEGACY_FROM_PSA_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_from_legacy.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_from_legacy.h deleted file mode 100644 index 14ca14696..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_from_legacy.h +++ /dev/null @@ -1,359 +0,0 @@ -/** - * \file mbedtls/config_adjust_psa_from_legacy.h - * \brief Adjust PSA configuration: construct PSA configuration from legacy - * - * This is an internal header. Do not include it directly. - * - * When MBEDTLS_PSA_CRYPTO_CONFIG is disabled, we automatically enable - * cryptographic mechanisms through the PSA interface when the corresponding - * legacy mechanism is enabled. In many cases, this just enables the PSA - * wrapper code around the legacy implementation, but we also do this for - * some mechanisms where PSA has its own independent implementation so - * that high-level modules that can use either cryptographic API have the - * same feature set in both cases. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_PSA_FROM_LEGACY_H -#define MBEDTLS_CONFIG_ADJUST_PSA_FROM_LEGACY_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/* - * Ensure PSA_WANT_* defines are setup properly if MBEDTLS_PSA_CRYPTO_CONFIG - * is not defined - */ - -#if defined(MBEDTLS_CCM_C) -#define MBEDTLS_PSA_BUILTIN_ALG_CCM 1 -#define PSA_WANT_ALG_CCM 1 -#if defined(MBEDTLS_CIPHER_C) -#define MBEDTLS_PSA_BUILTIN_ALG_CCM_STAR_NO_TAG 1 -#define PSA_WANT_ALG_CCM_STAR_NO_TAG 1 -#endif /* MBEDTLS_CIPHER_C */ -#endif /* MBEDTLS_CCM_C */ - -#if defined(MBEDTLS_CMAC_C) -#define MBEDTLS_PSA_BUILTIN_ALG_CMAC 1 -#define PSA_WANT_ALG_CMAC 1 -#endif /* MBEDTLS_CMAC_C */ - -#if defined(MBEDTLS_ECDH_C) -#define MBEDTLS_PSA_BUILTIN_ALG_ECDH 1 -#define PSA_WANT_ALG_ECDH 1 -#endif /* MBEDTLS_ECDH_C */ - -#if defined(MBEDTLS_ECDSA_C) -#define MBEDTLS_PSA_BUILTIN_ALG_ECDSA 1 -#define PSA_WANT_ALG_ECDSA 1 -#define PSA_WANT_ALG_ECDSA_ANY 1 - -// Only add in DETERMINISTIC support if ECDSA is also enabled -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) -#define MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA 1 -#define PSA_WANT_ALG_DETERMINISTIC_ECDSA 1 -#endif /* MBEDTLS_ECDSA_DETERMINISTIC */ - -#endif /* MBEDTLS_ECDSA_C */ - -#if defined(MBEDTLS_ECP_C) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE 1 -/* Normally we wouldn't enable this because it's not implemented in ecp.c, - * but since it used to be available any time ECP_C was enabled, let's enable - * it anyway for the sake of backwards compatibility */ -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_GENERATE 1 -/* See comment for PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE above. */ -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_KEY_PAIR_DERIVE 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ECC_PUBLIC_KEY 1 -#define PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY 1 -#endif /* MBEDTLS_ECP_C */ - -#if defined(MBEDTLS_DHM_C) -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE 1 -#define PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY 1 -#define PSA_WANT_ALG_FFDH 1 -#define PSA_WANT_DH_RFC7919_2048 1 -#define PSA_WANT_DH_RFC7919_3072 1 -#define PSA_WANT_DH_RFC7919_4096 1 -#define PSA_WANT_DH_RFC7919_6144 1 -#define PSA_WANT_DH_RFC7919_8192 1 -#define MBEDTLS_PSA_BUILTIN_ALG_FFDH 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_BASIC 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_IMPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_EXPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_KEY_PAIR_GENERATE 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DH_PUBLIC_KEY 1 -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_2048 1 -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_3072 1 -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_4096 1 -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_6144 1 -#define MBEDTLS_PSA_BUILTIN_DH_RFC7919_8192 1 -#endif /* MBEDTLS_DHM_C */ - -#if defined(MBEDTLS_GCM_C) -#define MBEDTLS_PSA_BUILTIN_ALG_GCM 1 -#define PSA_WANT_ALG_GCM 1 -#endif /* MBEDTLS_GCM_C */ - -/* Enable PSA HKDF algorithm if mbedtls HKDF is supported. - * PSA HKDF EXTRACT and PSA HKDF EXPAND have minimal cost when - * PSA HKDF is enabled, so enable both algorithms together - * with PSA HKDF. */ -#if defined(MBEDTLS_HKDF_C) -#define MBEDTLS_PSA_BUILTIN_ALG_HMAC 1 -#define PSA_WANT_ALG_HMAC 1 -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF 1 -#define PSA_WANT_ALG_HKDF 1 -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXTRACT 1 -#define PSA_WANT_ALG_HKDF_EXTRACT 1 -#define MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXPAND 1 -#define PSA_WANT_ALG_HKDF_EXPAND 1 -#endif /* MBEDTLS_HKDF_C */ - -#define MBEDTLS_PSA_BUILTIN_ALG_HMAC 1 -#define PSA_WANT_ALG_HMAC 1 -#define PSA_WANT_KEY_TYPE_HMAC 1 - -#if defined(MBEDTLS_MD_C) -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_PRF 1 -#define PSA_WANT_ALG_TLS12_PRF 1 -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS 1 -#define PSA_WANT_ALG_TLS12_PSK_TO_MS 1 -#endif /* MBEDTLS_MD_C */ - -#if defined(MBEDTLS_MD5_C) -#define MBEDTLS_PSA_BUILTIN_ALG_MD5 1 -#define PSA_WANT_ALG_MD5 1 -#endif - -#if defined(MBEDTLS_ECJPAKE_C) -#define MBEDTLS_PSA_BUILTIN_PAKE 1 -#define MBEDTLS_PSA_BUILTIN_ALG_JPAKE 1 -#define PSA_WANT_ALG_JPAKE 1 -#endif - -#if defined(MBEDTLS_RIPEMD160_C) -#define MBEDTLS_PSA_BUILTIN_ALG_RIPEMD160 1 -#define PSA_WANT_ALG_RIPEMD160 1 -#endif - -#if defined(MBEDTLS_RSA_C) -#if defined(MBEDTLS_PKCS1_V15) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PKCS1V15_CRYPT 1 -#define PSA_WANT_ALG_RSA_PKCS1V15_CRYPT 1 -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PKCS1V15_SIGN 1 -#define PSA_WANT_ALG_RSA_PKCS1V15_SIGN 1 -#define PSA_WANT_ALG_RSA_PKCS1V15_SIGN_RAW 1 -#endif /* MBEDTLS_PKCS1_V15 */ -#if defined(MBEDTLS_PKCS1_V21) -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_OAEP 1 -#define PSA_WANT_ALG_RSA_OAEP 1 -#define MBEDTLS_PSA_BUILTIN_ALG_RSA_PSS 1 -#define PSA_WANT_ALG_RSA_PSS 1 -#endif /* MBEDTLS_PKCS1_V21 */ -#if defined(MBEDTLS_GENPRIME) -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_GENERATE 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE 1 -#endif /* MBEDTLS_GENPRIME */ -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_RSA_PUBLIC_KEY 1 -#define PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY 1 -#endif /* MBEDTLS_RSA_C */ - -#if defined(MBEDTLS_SHA1_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_1 1 -#define PSA_WANT_ALG_SHA_1 1 -#endif - -#if defined(MBEDTLS_SHA224_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_224 1 -#define PSA_WANT_ALG_SHA_224 1 -#endif - -#if defined(MBEDTLS_SHA256_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_256 1 -#define PSA_WANT_ALG_SHA_256 1 -#endif - -#if defined(MBEDTLS_SHA384_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_384 1 -#define PSA_WANT_ALG_SHA_384 1 -#endif - -#if defined(MBEDTLS_SHA512_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA_512 1 -#define PSA_WANT_ALG_SHA_512 1 -#endif - -#if defined(MBEDTLS_SHA3_C) -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_224 1 -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_256 1 -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_384 1 -#define MBEDTLS_PSA_BUILTIN_ALG_SHA3_512 1 -#define PSA_WANT_ALG_SHA3_224 1 -#define PSA_WANT_ALG_SHA3_256 1 -#define PSA_WANT_ALG_SHA3_384 1 -#define PSA_WANT_ALG_SHA3_512 1 -#endif - -#if defined(MBEDTLS_AES_C) -#define PSA_WANT_KEY_TYPE_AES 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_AES 1 -#endif - -#if defined(MBEDTLS_ARIA_C) -#define PSA_WANT_KEY_TYPE_ARIA 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_ARIA 1 -#endif - -#if defined(MBEDTLS_CAMELLIA_C) -#define PSA_WANT_KEY_TYPE_CAMELLIA 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_CAMELLIA 1 -#endif - -#if defined(MBEDTLS_DES_C) -#define PSA_WANT_KEY_TYPE_DES 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_DES 1 -#endif - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_256) -#define MBEDTLS_PSA_BUILTIN_ALG_TLS12_ECJPAKE_TO_PMS 1 -#define PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS 1 -#endif - -#if defined(MBEDTLS_CHACHA20_C) -#define PSA_WANT_KEY_TYPE_CHACHA20 1 -#define MBEDTLS_PSA_BUILTIN_KEY_TYPE_CHACHA20 1 -/* ALG_STREAM_CIPHER requires CIPHER_C in order to be supported in PSA */ -#if defined(MBEDTLS_CIPHER_C) -#define PSA_WANT_ALG_STREAM_CIPHER 1 -#define MBEDTLS_PSA_BUILTIN_ALG_STREAM_CIPHER 1 -#endif -#if defined(MBEDTLS_CHACHAPOLY_C) -#define PSA_WANT_ALG_CHACHA20_POLY1305 1 -#define MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305 1 -#endif -#endif - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -#define MBEDTLS_PSA_BUILTIN_ALG_CBC_NO_PADDING 1 -#define PSA_WANT_ALG_CBC_NO_PADDING 1 -#if defined(MBEDTLS_CIPHER_PADDING_PKCS7) -#define MBEDTLS_PSA_BUILTIN_ALG_CBC_PKCS7 1 -#define PSA_WANT_ALG_CBC_PKCS7 1 -#endif -#endif - -#if (defined(MBEDTLS_AES_C) || defined(MBEDTLS_DES_C) || \ - defined(MBEDTLS_ARIA_C) || defined(MBEDTLS_CAMELLIA_C)) && \ - defined(MBEDTLS_CIPHER_C) -#define MBEDTLS_PSA_BUILTIN_ALG_ECB_NO_PADDING 1 -#define PSA_WANT_ALG_ECB_NO_PADDING 1 -#endif - -#if defined(MBEDTLS_CIPHER_MODE_CFB) -#define MBEDTLS_PSA_BUILTIN_ALG_CFB 1 -#define PSA_WANT_ALG_CFB 1 -#endif - -#if defined(MBEDTLS_CIPHER_MODE_CTR) -#define MBEDTLS_PSA_BUILTIN_ALG_CTR 1 -#define PSA_WANT_ALG_CTR 1 -#endif - -#if defined(MBEDTLS_CIPHER_MODE_OFB) -#define MBEDTLS_PSA_BUILTIN_ALG_OFB 1 -#define PSA_WANT_ALG_OFB 1 -#endif - -#if defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_256 1 -#define PSA_WANT_ECC_BRAINPOOL_P_R1_256 1 -#endif - -#if defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_384 1 -#define PSA_WANT_ECC_BRAINPOOL_P_R1_384 1 -#endif - -#if defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_BRAINPOOL_P_R1_512 1 -#define PSA_WANT_ECC_BRAINPOOL_P_R1_512 1 -#endif - -#if defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_MONTGOMERY_255 1 -#define PSA_WANT_ECC_MONTGOMERY_255 1 -#endif - -#if defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_MONTGOMERY_448 1 -#define PSA_WANT_ECC_MONTGOMERY_448 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_192 1 -#define PSA_WANT_ECC_SECP_R1_192 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_224 1 -#define PSA_WANT_ECC_SECP_R1_224 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_256 1 -#define PSA_WANT_ECC_SECP_R1_256 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_384 1 -#define PSA_WANT_ECC_SECP_R1_384 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_R1_521 1 -#define PSA_WANT_ECC_SECP_R1_521 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_K1_192 1 -#define PSA_WANT_ECC_SECP_K1_192 1 -#endif - -/* SECP224K1 is buggy via the PSA API (https://github.com/Mbed-TLS/mbedtls/issues/3541) */ -#if 0 && defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_K1_224 1 -#define PSA_WANT_ECC_SECP_K1_224 1 -#endif - -#if defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) -#define MBEDTLS_PSA_BUILTIN_ECC_SECP_K1_256 1 -#define PSA_WANT_ECC_SECP_K1_256 1 -#endif - -#endif /* MBEDTLS_CONFIG_ADJUST_PSA_FROM_LEGACY_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_superset_legacy.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_superset_legacy.h deleted file mode 100644 index 1a232cbb8..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_psa_superset_legacy.h +++ /dev/null @@ -1,145 +0,0 @@ -/** - * \file mbedtls/config_adjust_psa_superset_legacy.h - * \brief Adjust PSA configuration: automatic enablement from legacy - * - * This is an internal header. Do not include it directly. - * - * To simplify some edge cases, we automatically enable certain cryptographic - * mechanisms in the PSA API if they are enabled in the legacy API. The general - * idea is that if legacy module M uses mechanism A internally, and A has - * both a legacy and a PSA implementation, we enable A through PSA whenever - * it's enabled through legacy. This facilitates the transition to PSA - * implementations of A for users of M. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_PSA_SUPERSET_LEGACY_H -#define MBEDTLS_CONFIG_ADJUST_PSA_SUPERSET_LEGACY_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/****************************************************************/ -/* Hashes that are built in are also enabled in PSA. - * This simplifies dependency declarations especially - * for modules that obey MBEDTLS_USE_PSA_CRYPTO. */ -/****************************************************************/ - -#if defined(MBEDTLS_MD5_C) -#define PSA_WANT_ALG_MD5 1 -#endif - -#if defined(MBEDTLS_RIPEMD160_C) -#define PSA_WANT_ALG_RIPEMD160 1 -#endif - -#if defined(MBEDTLS_SHA1_C) -#define PSA_WANT_ALG_SHA_1 1 -#endif - -#if defined(MBEDTLS_SHA224_C) -#define PSA_WANT_ALG_SHA_224 1 -#endif - -#if defined(MBEDTLS_SHA256_C) -#define PSA_WANT_ALG_SHA_256 1 -#endif - -#if defined(MBEDTLS_SHA384_C) -#define PSA_WANT_ALG_SHA_384 1 -#endif - -#if defined(MBEDTLS_SHA512_C) -#define PSA_WANT_ALG_SHA_512 1 -#endif - -#if defined(MBEDTLS_SHA3_C) -#define PSA_WANT_ALG_SHA3_224 1 -#define PSA_WANT_ALG_SHA3_256 1 -#define PSA_WANT_ALG_SHA3_384 1 -#define PSA_WANT_ALG_SHA3_512 1 -#endif - -/* Ensure that the PSA's supported curves (PSA_WANT_ECC_xxx) are always a - * superset of the builtin ones (MBEDTLS_ECP_DP_xxx). */ -#if defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) -#if !defined(PSA_WANT_ECC_BRAINPOOL_P_R1_256) -#define PSA_WANT_ECC_BRAINPOOL_P_R1_256 1 -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_256 */ -#endif /* MBEDTLS_ECP_DP_BP256R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) -#if !defined(PSA_WANT_ECC_BRAINPOOL_P_R1_384) -#define PSA_WANT_ECC_BRAINPOOL_P_R1_384 1 -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_384 */ -#endif /*MBEDTLS_ECP_DP_BP384R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) -#if !defined(PSA_WANT_ECC_BRAINPOOL_P_R1_512) -#define PSA_WANT_ECC_BRAINPOOL_P_R1_512 1 -#endif /* PSA_WANT_ECC_BRAINPOOL_P_R1_512 */ -#endif /* MBEDTLS_ECP_DP_BP512R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) -#if !defined(PSA_WANT_ECC_MONTGOMERY_255) -#define PSA_WANT_ECC_MONTGOMERY_255 1 -#endif /* PSA_WANT_ECC_MONTGOMERY_255 */ -#endif /* MBEDTLS_ECP_DP_CURVE25519_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) -#if !defined(PSA_WANT_ECC_MONTGOMERY_448) -#define PSA_WANT_ECC_MONTGOMERY_448 1 -#endif /* PSA_WANT_ECC_MONTGOMERY_448 */ -#endif /* MBEDTLS_ECP_DP_CURVE448_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_R1_192) -#define PSA_WANT_ECC_SECP_R1_192 1 -#endif /* PSA_WANT_ECC_SECP_R1_192 */ -#endif /* MBEDTLS_ECP_DP_SECP192R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_R1_224) -#define PSA_WANT_ECC_SECP_R1_224 1 -#endif /* PSA_WANT_ECC_SECP_R1_224 */ -#endif /* MBEDTLS_ECP_DP_SECP224R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_R1_256) -#define PSA_WANT_ECC_SECP_R1_256 1 -#endif /* PSA_WANT_ECC_SECP_R1_256 */ -#endif /* MBEDTLS_ECP_DP_SECP256R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_R1_384) -#define PSA_WANT_ECC_SECP_R1_384 1 -#endif /* PSA_WANT_ECC_SECP_R1_384 */ -#endif /* MBEDTLS_ECP_DP_SECP384R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_R1_521) -#define PSA_WANT_ECC_SECP_R1_521 1 -#endif /* PSA_WANT_ECC_SECP_R1_521 */ -#endif /* MBEDTLS_ECP_DP_SECP521R1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_K1_192) -#define PSA_WANT_ECC_SECP_K1_192 1 -#endif /* PSA_WANT_ECC_SECP_K1_192 */ -#endif /* MBEDTLS_ECP_DP_SECP192K1_ENABLED */ - -#if defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) -#if !defined(PSA_WANT_ECC_SECP_K1_256) -#define PSA_WANT_ECC_SECP_K1_256 1 -#endif /* PSA_WANT_ECC_SECP_K1_256 */ -#endif /* MBEDTLS_ECP_DP_SECP256K1_ENABLED */ - -#endif /* MBEDTLS_CONFIG_ADJUST_PSA_SUPERSET_LEGACY_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_ssl.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_ssl.h deleted file mode 100644 index 1f82d9c00..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_ssl.h +++ /dev/null @@ -1,91 +0,0 @@ -/** - * \file mbedtls/config_adjust_ssl.h - * \brief Adjust TLS configuration - * - * This is an internal header. Do not include it directly. - * - * Automatically enable certain dependencies. Generally, MBEDTLS_xxx - * configurations need to be explicitly enabled by the user: enabling - * MBEDTLS_xxx_A but not MBEDTLS_xxx_B when A requires B results in a - * compilation error. However, we do automatically enable certain options - * in some circumstances. One case is if MBEDTLS_xxx_B is an internal option - * used to identify parts of a module that are used by other module, and we - * don't want to make the symbol MBEDTLS_xxx_B part of the public API. - * Another case is if A didn't depend on B in earlier versions, and we - * want to use B in A but we need to preserve backward compatibility with - * configurations that explicitly activate MBEDTLS_xxx_A but not - * MBEDTLS_xxx_B. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_SSL_H -#define MBEDTLS_CONFIG_ADJUST_SSL_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/* The following blocks make it easier to disable all of TLS, - * or of TLS 1.2 or 1.3 or DTLS, without having to manually disable all - * key exchanges, options and extensions related to them. */ - -#if !defined(MBEDTLS_SSL_TLS_C) -#undef MBEDTLS_SSL_CLI_C -#undef MBEDTLS_SSL_SRV_C -#undef MBEDTLS_SSL_PROTO_TLS1_3 -#undef MBEDTLS_SSL_PROTO_TLS1_2 -#undef MBEDTLS_SSL_PROTO_DTLS -#endif - -#if !(defined(MBEDTLS_SSL_SRV_C) && defined(MBEDTLS_SSL_SESSION_TICKETS)) -#undef MBEDTLS_SSL_TICKET_C -#endif - -#if !defined(MBEDTLS_SSL_PROTO_DTLS) -#undef MBEDTLS_SSL_DTLS_ANTI_REPLAY -#undef MBEDTLS_SSL_DTLS_CONNECTION_ID -#undef MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT -#undef MBEDTLS_SSL_DTLS_HELLO_VERIFY -#undef MBEDTLS_SSL_DTLS_SRTP -#undef MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE -#endif - -#if !defined(MBEDTLS_SSL_PROTO_TLS1_2) -#undef MBEDTLS_SSL_ENCRYPT_THEN_MAC -#undef MBEDTLS_SSL_EXTENDED_MASTER_SECRET -#undef MBEDTLS_SSL_RENEGOTIATION -#undef MBEDTLS_KEY_EXCHANGE_RSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_PSK_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED -#undef MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED -#endif - -#if !defined(MBEDTLS_SSL_PROTO_TLS1_3) -#undef MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED -#undef MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED -#undef MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED -#undef MBEDTLS_SSL_EARLY_DATA -#undef MBEDTLS_SSL_RECORD_SIZE_LIMIT -#endif - -#if defined(MBEDTLS_SSL_PROTO_TLS1_2) && \ - (defined(MBEDTLS_ECDH_C) || defined(MBEDTLS_ECDSA_C) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED)) -#define MBEDTLS_SSL_TLS1_2_SOME_ECC -#endif - -#endif /* MBEDTLS_CONFIG_ADJUST_SSL_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_x509.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_x509.h deleted file mode 100644 index cfb2d8891..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_adjust_x509.h +++ /dev/null @@ -1,35 +0,0 @@ -/** - * \file mbedtls/config_adjust_x509.h - * \brief Adjust X.509 configuration - * - * This is an internal header. Do not include it directly. - * - * Automatically enable certain dependencies. Generally, MBEDTLS_xxx - * configurations need to be explicitly enabled by the user: enabling - * MBEDTLS_xxx_A but not MBEDTLS_xxx_B when A requires B results in a - * compilation error. However, we do automatically enable certain options - * in some circumstances. One case is if MBEDTLS_xxx_B is an internal option - * used to identify parts of a module that are used by other module, and we - * don't want to make the symbol MBEDTLS_xxx_B part of the public API. - * Another case is if A didn't depend on B in earlier versions, and we - * want to use B in A but we need to preserve backward compatibility with - * configurations that explicitly activate MBEDTLS_xxx_A but not - * MBEDTLS_xxx_B. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_ADJUST_X509_H -#define MBEDTLS_CONFIG_ADJUST_X509_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include mbedtls/config_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -#endif /* MBEDTLS_CONFIG_ADJUST_X509_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/config_psa.h b/android/mbedtls/arm64-v8a/include/mbedtls/config_psa.h deleted file mode 100644 index 5f3d0f3d5..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/config_psa.h +++ /dev/null @@ -1,61 +0,0 @@ -/** - * \file mbedtls/config_psa.h - * \brief PSA crypto configuration options (set of defines) - * - * This set of compile-time options takes settings defined in - * include/mbedtls/mbedtls_config.h and include/psa/crypto_config.h and uses - * those definitions to define symbols used in the library code. - * - * Users and integrators should not edit this file, please edit - * include/mbedtls/mbedtls_config.h for MBEDTLS_XXX settings or - * include/psa/crypto_config.h for PSA_WANT_XXX settings. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONFIG_PSA_H -#define MBEDTLS_CONFIG_PSA_H - -#include "psa/crypto_legacy.h" - -#include "psa/crypto_adjust_config_synonyms.h" - -#include "psa/crypto_adjust_config_dependencies.h" - -#include "mbedtls/config_adjust_psa_superset_legacy.h" - -#if defined(MBEDTLS_PSA_CRYPTO_CONFIG) - -/* Require built-in implementations based on PSA requirements */ - -/* We need this to have a complete list of requirements - * before we deduce what built-ins are required. */ -#include "psa/crypto_adjust_config_key_pair_types.h" - -#if defined(MBEDTLS_PSA_CRYPTO_C) -/* If we are implementing PSA crypto ourselves, then we want to enable the - * required built-ins. Otherwise, PSA features will be provided by the server. */ -#include "mbedtls/config_adjust_legacy_from_psa.h" -#endif - -#else /* MBEDTLS_PSA_CRYPTO_CONFIG */ - -/* Infer PSA requirements from Mbed TLS capabilities */ - -#include "mbedtls/config_adjust_psa_from_legacy.h" - -/* Hopefully the file above will have enabled keypair symbols in a consistent - * way, but including this here fixes them if that wasn't the case. */ -#include "psa/crypto_adjust_config_key_pair_types.h" - -#endif /* MBEDTLS_PSA_CRYPTO_CONFIG */ - -#if defined(PSA_WANT_ALG_JPAKE) -#define PSA_WANT_ALG_SOME_PAKE 1 -#endif - -#include "psa/crypto_adjust_auto_enabled.h" - -#endif /* MBEDTLS_CONFIG_PSA_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/constant_time.h b/android/mbedtls/arm64-v8a/include/mbedtls/constant_time.h deleted file mode 100644 index d31bff677..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/constant_time.h +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Constant-time functions - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CONSTANT_TIME_H -#define MBEDTLS_CONSTANT_TIME_H - -#include - -/** Constant-time buffer comparison without branches. - * - * This is equivalent to the standard memcmp function, but is likely to be - * compiled to code using bitwise operations rather than a branch, such that - * the time taken is constant w.r.t. the data pointed to by \p a and \p b, - * and w.r.t. whether \p a and \p b are equal or not. It is not constant-time - * w.r.t. \p n . - * - * This function can be used to write constant-time code by replacing branches - * with bit operations using masks. - * - * \param a Pointer to the first buffer, containing at least \p n bytes. May not be NULL. - * \param b Pointer to the second buffer, containing at least \p n bytes. May not be NULL. - * \param n The number of bytes to compare. - * - * \return Zero if the contents of the two buffers are the same, - * otherwise non-zero. - */ -int mbedtls_ct_memcmp(const void *a, - const void *b, - size_t n); - -#endif /* MBEDTLS_CONSTANT_TIME_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ctr_drbg.h b/android/mbedtls/arm64-v8a/include/mbedtls/ctr_drbg.h deleted file mode 100644 index 0b7cce192..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ctr_drbg.h +++ /dev/null @@ -1,597 +0,0 @@ -/** - * \file ctr_drbg.h - * - * \brief This file contains definitions and functions for the - * CTR_DRBG pseudorandom generator. - * - * CTR_DRBG is a standardized way of building a PRNG from a block-cipher - * in counter mode operation, as defined in NIST SP 800-90A: - * Recommendation for Random Number Generation Using Deterministic Random - * Bit Generators. - * - * The Mbed TLS implementation of CTR_DRBG uses AES-256 (default) or AES-128 - * (if \c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY is enabled at compile time) - * as the underlying block cipher, with a derivation function. - * - * The security strength as defined in NIST SP 800-90A is - * 128 bits when AES-128 is used (\c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY enabled) - * and 256 bits otherwise, provided that #MBEDTLS_CTR_DRBG_ENTROPY_LEN is - * kept at its default value (and not overridden in mbedtls_config.h) and that the - * DRBG instance is set up with default parameters. - * See the documentation of mbedtls_ctr_drbg_seed() for more - * information. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_CTR_DRBG_H -#define MBEDTLS_CTR_DRBG_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -/* The CTR_DRBG implementation can either directly call the low-level AES - * module (gated by MBEDTLS_AES_C) or call the PSA API to perform AES - * operations. Calling the AES module directly is the default, both for - * maximum backward compatibility and because it's a bit more efficient - * (less glue code). - * - * When MBEDTLS_AES_C is disabled, the CTR_DRBG module calls PSA crypto and - * thus benefits from the PSA AES accelerator driver. - * It is technically possible to enable MBEDTLS_CTR_DRBG_USE_PSA_CRYPTO - * to use PSA even when MBEDTLS_AES_C is enabled, but there is very little - * reason to do so other than testing purposes and this is not officially - * supported. - */ -#if !defined(MBEDTLS_AES_C) -#define MBEDTLS_CTR_DRBG_USE_PSA_CRYPTO -#endif - -#if defined(MBEDTLS_CTR_DRBG_USE_PSA_CRYPTO) -#include "psa/crypto.h" -#else -#include "mbedtls/aes.h" -#endif - -#include "entropy.h" - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -/** The entropy source failed. */ -#define MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED -0x0034 -/** The requested random buffer length is too big. */ -#define MBEDTLS_ERR_CTR_DRBG_REQUEST_TOO_BIG -0x0036 -/** The input (entropy + additional data) is too large. */ -#define MBEDTLS_ERR_CTR_DRBG_INPUT_TOO_BIG -0x0038 -/** Read or write error in file. */ -#define MBEDTLS_ERR_CTR_DRBG_FILE_IO_ERROR -0x003A - -#define MBEDTLS_CTR_DRBG_BLOCKSIZE 16 /**< The block size used by the cipher. */ - -#if defined(MBEDTLS_CTR_DRBG_USE_128_BIT_KEY) -#define MBEDTLS_CTR_DRBG_KEYSIZE 16 -/**< The key size in bytes used by the cipher. - * - * Compile-time choice: 16 bytes (128 bits) - * because #MBEDTLS_CTR_DRBG_USE_128_BIT_KEY is enabled. - */ -#else -#define MBEDTLS_CTR_DRBG_KEYSIZE 32 -/**< The key size in bytes used by the cipher. - * - * Compile-time choice: 32 bytes (256 bits) - * because \c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY is disabled. - */ -#endif - -#define MBEDTLS_CTR_DRBG_KEYBITS (MBEDTLS_CTR_DRBG_KEYSIZE * 8) /**< The key size for the DRBG operation, in bits. */ -#define MBEDTLS_CTR_DRBG_SEEDLEN (MBEDTLS_CTR_DRBG_KEYSIZE + MBEDTLS_CTR_DRBG_BLOCKSIZE) /**< The seed length, calculated as (counter + AES key). */ - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them using the compiler command - * line. - * \{ - */ - -/** \def MBEDTLS_CTR_DRBG_ENTROPY_LEN - * - * \brief The amount of entropy used per seed by default, in bytes. - */ -#if !defined(MBEDTLS_CTR_DRBG_ENTROPY_LEN) -#if defined(MBEDTLS_ENTROPY_SHA512_ACCUMULATOR) -/** This is 48 bytes because the entropy module uses SHA-512. - */ -#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 - -#else /* MBEDTLS_ENTROPY_SHA512_ACCUMULATOR */ - -/** This is 32 bytes because the entropy module uses SHA-256. - */ -#if !defined(MBEDTLS_CTR_DRBG_USE_128_BIT_KEY) -/** \warning To achieve a 256-bit security strength, you must pass a nonce - * to mbedtls_ctr_drbg_seed(). - */ -#endif /* !defined(MBEDTLS_CTR_DRBG_USE_128_BIT_KEY) */ -#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 32 -#endif /* MBEDTLS_ENTROPY_SHA512_ACCUMULATOR */ -#endif /* !defined(MBEDTLS_CTR_DRBG_ENTROPY_LEN) */ - -#if !defined(MBEDTLS_CTR_DRBG_RESEED_INTERVAL) -#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 -/**< The interval before reseed is performed by default. */ -#endif - -#if !defined(MBEDTLS_CTR_DRBG_MAX_INPUT) -#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 -/**< The maximum number of additional input Bytes. */ -#endif - -#if !defined(MBEDTLS_CTR_DRBG_MAX_REQUEST) -#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 -/**< The maximum number of requested Bytes per call. */ -#endif - -#if !defined(MBEDTLS_CTR_DRBG_MAX_SEED_INPUT) -#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 -/**< The maximum size of seed or reseed buffer in bytes. */ -#endif - -/** \} name SECTION: Module settings */ - -#define MBEDTLS_CTR_DRBG_PR_OFF 0 -/**< Prediction resistance is disabled. */ -#define MBEDTLS_CTR_DRBG_PR_ON 1 -/**< Prediction resistance is enabled. */ - -#ifdef __cplusplus -extern "C" { -#endif - -#if MBEDTLS_CTR_DRBG_ENTROPY_LEN >= MBEDTLS_CTR_DRBG_KEYSIZE * 3 / 2 -/** The default length of the nonce read from the entropy source. - * - * This is \c 0 because a single read from the entropy source is sufficient - * to include a nonce. - * See the documentation of mbedtls_ctr_drbg_seed() for more information. - */ -#define MBEDTLS_CTR_DRBG_ENTROPY_NONCE_LEN 0 -#else -/** The default length of the nonce read from the entropy source. - * - * This is half of the default entropy length because a single read from - * the entropy source does not provide enough material to form a nonce. - * See the documentation of mbedtls_ctr_drbg_seed() for more information. - */ -#define MBEDTLS_CTR_DRBG_ENTROPY_NONCE_LEN (MBEDTLS_CTR_DRBG_ENTROPY_LEN + 1) / 2 -#endif - -#if defined(MBEDTLS_CTR_DRBG_USE_PSA_CRYPTO) -typedef struct mbedtls_ctr_drbg_psa_context { - mbedtls_svc_key_id_t key_id; - psa_cipher_operation_t operation; -} mbedtls_ctr_drbg_psa_context; -#endif - -/** - * \brief The CTR_DRBG context structure. - */ -typedef struct mbedtls_ctr_drbg_context { - unsigned char MBEDTLS_PRIVATE(counter)[16]; /*!< The counter (V). */ - int MBEDTLS_PRIVATE(reseed_counter); /*!< The reseed counter. - * This is the number of requests that have - * been made since the last (re)seeding, - * minus one. - * Before the initial seeding, this field - * contains the amount of entropy in bytes - * to use as a nonce for the initial seeding, - * or -1 if no nonce length has been explicitly - * set (see mbedtls_ctr_drbg_set_nonce_len()). - */ - int MBEDTLS_PRIVATE(prediction_resistance); /*!< This determines whether prediction - resistance is enabled, that is - whether to systematically reseed before - each random generation. */ - size_t MBEDTLS_PRIVATE(entropy_len); /*!< The amount of entropy grabbed on each - seed or reseed operation, in bytes. */ - int MBEDTLS_PRIVATE(reseed_interval); /*!< The reseed interval. - * This is the maximum number of requests - * that can be made between reseedings. */ - -#if defined(MBEDTLS_CTR_DRBG_USE_PSA_CRYPTO) - mbedtls_ctr_drbg_psa_context MBEDTLS_PRIVATE(psa_ctx); /*!< The PSA context. */ -#else - mbedtls_aes_context MBEDTLS_PRIVATE(aes_ctx); /*!< The AES context. */ -#endif - - /* - * Callbacks (Entropy) - */ - int(*MBEDTLS_PRIVATE(f_entropy))(void *, unsigned char *, size_t); - /*!< The entropy callback function. */ - - void *MBEDTLS_PRIVATE(p_entropy); /*!< The context for the entropy function. */ - -#if defined(MBEDTLS_THREADING_C) - /* Invariant: the mutex is initialized if and only if f_entropy != NULL. - * This means that the mutex is initialized during the initial seeding - * in mbedtls_ctr_drbg_seed() and freed in mbedtls_ctr_drbg_free(). - * - * Note that this invariant may change without notice. Do not rely on it - * and do not access the mutex directly in application code. - */ - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); -#endif -} -mbedtls_ctr_drbg_context; - -/** - * \brief This function initializes the CTR_DRBG context, - * and prepares it for mbedtls_ctr_drbg_seed() - * or mbedtls_ctr_drbg_free(). - * - * \note The reseed interval is - * #MBEDTLS_CTR_DRBG_RESEED_INTERVAL by default. - * You can override it by calling - * mbedtls_ctr_drbg_set_reseed_interval(). - * - * \param ctx The CTR_DRBG context to initialize. - */ -void mbedtls_ctr_drbg_init(mbedtls_ctr_drbg_context *ctx); - -/** - * \brief This function seeds and sets up the CTR_DRBG - * entropy source for future reseeds. - * - * A typical choice for the \p f_entropy and \p p_entropy parameters is - * to use the entropy module: - * - \p f_entropy is mbedtls_entropy_func(); - * - \p p_entropy is an instance of ::mbedtls_entropy_context initialized - * with mbedtls_entropy_init() (which registers the platform's default - * entropy sources). - * - * The entropy length is #MBEDTLS_CTR_DRBG_ENTROPY_LEN by default. - * You can override it by calling mbedtls_ctr_drbg_set_entropy_len(). - * - * The entropy nonce length is: - * - \c 0 if the entropy length is at least 3/2 times the entropy length, - * which guarantees that the security strength is the maximum permitted - * by the key size and entropy length according to NIST SP 800-90A §10.2.1; - * - Half the entropy length otherwise. - * You can override it by calling mbedtls_ctr_drbg_set_nonce_len(). - * With the default entropy length, the entropy nonce length is - * #MBEDTLS_CTR_DRBG_ENTROPY_NONCE_LEN. - * - * You can provide a nonce and personalization string in addition to the - * entropy source, to make this instantiation as unique as possible. - * See SP 800-90A §8.6.7 for more details about nonces. - * - * The _seed_material_ value passed to the derivation function in - * the CTR_DRBG Instantiate Process described in NIST SP 800-90A §10.2.1.3.2 - * is the concatenation of the following strings: - * - A string obtained by calling \p f_entropy function for the entropy - * length. - */ -#if MBEDTLS_CTR_DRBG_ENTROPY_NONCE_LEN == 0 -/** - * - If mbedtls_ctr_drbg_set_nonce_len() has been called, a string - * obtained by calling \p f_entropy function for the specified length. - */ -#else -/** - * - A string obtained by calling \p f_entropy function for the entropy nonce - * length. If the entropy nonce length is \c 0, this function does not - * make a second call to \p f_entropy. - */ -#endif -#if defined(MBEDTLS_THREADING_C) -/** - * \note When Mbed TLS is built with threading support, - * after this function returns successfully, - * it is safe to call mbedtls_ctr_drbg_random() - * from multiple threads. Other operations, including - * reseeding, are not thread-safe. - */ -#endif /* MBEDTLS_THREADING_C */ -/** - * - The \p custom string. - * - * \note To achieve the nominal security strength permitted - * by CTR_DRBG, the entropy length must be: - * - at least 16 bytes for a 128-bit strength - * (maximum achievable strength when using AES-128); - * - at least 32 bytes for a 256-bit strength - * (maximum achievable strength when using AES-256). - * - * In addition, if you do not pass a nonce in \p custom, - * the sum of the entropy length - * and the entropy nonce length must be: - * - at least 24 bytes for a 128-bit strength - * (maximum achievable strength when using AES-128); - * - at least 48 bytes for a 256-bit strength - * (maximum achievable strength when using AES-256). - * - * \param ctx The CTR_DRBG context to seed. - * It must have been initialized with - * mbedtls_ctr_drbg_init(). - * After a successful call to mbedtls_ctr_drbg_seed(), - * you may not call mbedtls_ctr_drbg_seed() again on - * the same context unless you call - * mbedtls_ctr_drbg_free() and mbedtls_ctr_drbg_init() - * again first. - * After a failed call to mbedtls_ctr_drbg_seed(), - * you must call mbedtls_ctr_drbg_free(). - * \param f_entropy The entropy callback, taking as arguments the - * \p p_entropy context, the buffer to fill, and the - * length of the buffer. - * \p f_entropy is always called with a buffer size - * less than or equal to the entropy length. - * \param p_entropy The entropy context to pass to \p f_entropy. - * \param custom The personalization string. - * This can be \c NULL, in which case the personalization - * string is empty regardless of the value of \p len. - * \param len The length of the personalization string. - * This must be at most - * #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT - * - #MBEDTLS_CTR_DRBG_ENTROPY_LEN. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED on failure. - */ -int mbedtls_ctr_drbg_seed(mbedtls_ctr_drbg_context *ctx, - int (*f_entropy)(void *, unsigned char *, size_t), - void *p_entropy, - const unsigned char *custom, - size_t len); - -/** - * \brief This function resets CTR_DRBG context to the state immediately - * after initial call of mbedtls_ctr_drbg_init(). - * - * \param ctx The CTR_DRBG context to clear. - */ -void mbedtls_ctr_drbg_free(mbedtls_ctr_drbg_context *ctx); - -/** - * \brief This function turns prediction resistance on or off. - * The default value is off. - * - * \note If enabled, entropy is gathered at the beginning of - * every call to mbedtls_ctr_drbg_random_with_add() - * or mbedtls_ctr_drbg_random(). - * Only use this if your entropy source has sufficient - * throughput. - * - * \param ctx The CTR_DRBG context. - * \param resistance #MBEDTLS_CTR_DRBG_PR_ON or #MBEDTLS_CTR_DRBG_PR_OFF. - */ -void mbedtls_ctr_drbg_set_prediction_resistance(mbedtls_ctr_drbg_context *ctx, - int resistance); - -/** - * \brief This function sets the amount of entropy grabbed on each - * seed or reseed. - * - * The default value is #MBEDTLS_CTR_DRBG_ENTROPY_LEN. - * - * \note The security strength of CTR_DRBG is bounded by the - * entropy length. Thus: - * - When using AES-256 - * (\c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY is disabled, - * which is the default), - * \p len must be at least 32 (in bytes) - * to achieve a 256-bit strength. - * - When using AES-128 - * (\c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY is enabled) - * \p len must be at least 16 (in bytes) - * to achieve a 128-bit strength. - * - * \param ctx The CTR_DRBG context. - * \param len The amount of entropy to grab, in bytes. - * This must be at most #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT - * and at most the maximum length accepted by the - * entropy function that is set in the context. - */ -void mbedtls_ctr_drbg_set_entropy_len(mbedtls_ctr_drbg_context *ctx, - size_t len); - -/** - * \brief This function sets the amount of entropy grabbed - * as a nonce for the initial seeding. - * - * Call this function before calling mbedtls_ctr_drbg_seed() to read - * a nonce from the entropy source during the initial seeding. - * - * \param ctx The CTR_DRBG context. - * \param len The amount of entropy to grab for the nonce, in bytes. - * This must be at most #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT - * and at most the maximum length accepted by the - * entropy function that is set in the context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_INPUT_TOO_BIG if \p len is - * more than #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED - * if the initial seeding has already taken place. - */ -int mbedtls_ctr_drbg_set_nonce_len(mbedtls_ctr_drbg_context *ctx, - size_t len); - -/** - * \brief This function sets the reseed interval. - * - * The reseed interval is the number of calls to mbedtls_ctr_drbg_random() - * or mbedtls_ctr_drbg_random_with_add() after which the entropy function - * is called again. - * - * The default value is #MBEDTLS_CTR_DRBG_RESEED_INTERVAL. - * - * \param ctx The CTR_DRBG context. - * \param interval The reseed interval. - */ -void mbedtls_ctr_drbg_set_reseed_interval(mbedtls_ctr_drbg_context *ctx, - int interval); - -/** - * \brief This function reseeds the CTR_DRBG context, that is - * extracts data from the entropy source. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param ctx The CTR_DRBG context. - * \param additional Additional data to add to the state. Can be \c NULL. - * \param len The length of the additional data. - * This must be less than - * #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT - \c entropy_len - * where \c entropy_len is the entropy length - * configured for the context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED on failure. - */ -int mbedtls_ctr_drbg_reseed(mbedtls_ctr_drbg_context *ctx, - const unsigned char *additional, size_t len); - -/** - * \brief This function updates the state of the CTR_DRBG context. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param ctx The CTR_DRBG context. - * \param additional The data to update the state with. This must not be - * \c NULL unless \p add_len is \c 0. - * \param add_len Length of \p additional in bytes. This must be at - * most #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_INPUT_TOO_BIG if - * \p add_len is more than - * #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT. - * \return An error from the underlying AES cipher on failure. - */ -int mbedtls_ctr_drbg_update(mbedtls_ctr_drbg_context *ctx, - const unsigned char *additional, - size_t add_len); - -/** - * \brief This function updates a CTR_DRBG instance with additional - * data and uses it to generate random data. - * - * This function automatically reseeds if the reseed counter is exceeded - * or prediction resistance is enabled. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param p_rng The CTR_DRBG context. This must be a pointer to a - * #mbedtls_ctr_drbg_context structure. - * \param output The buffer to fill. - * \param output_len The length of the buffer in bytes. - * \param additional Additional data to update. Can be \c NULL, in which - * case the additional data is empty regardless of - * the value of \p add_len. - * \param add_len The length of the additional data - * if \p additional is not \c NULL. - * This must be less than #MBEDTLS_CTR_DRBG_MAX_INPUT - * and less than - * #MBEDTLS_CTR_DRBG_MAX_SEED_INPUT - \c entropy_len - * where \c entropy_len is the entropy length - * configured for the context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED or - * #MBEDTLS_ERR_CTR_DRBG_REQUEST_TOO_BIG on failure. - */ -int mbedtls_ctr_drbg_random_with_add(void *p_rng, - unsigned char *output, size_t output_len, - const unsigned char *additional, size_t add_len); - -/** - * \brief This function uses CTR_DRBG to generate random data. - * - * This function automatically reseeds if the reseed counter is exceeded - * or prediction resistance is enabled. - */ -#if defined(MBEDTLS_THREADING_C) -/** - * \note When Mbed TLS is built with threading support, - * it is safe to call mbedtls_ctr_drbg_random() - * from multiple threads. Other operations, including - * reseeding, are not thread-safe. - */ -#endif /* MBEDTLS_THREADING_C */ -/** - * \param p_rng The CTR_DRBG context. This must be a pointer to a - * #mbedtls_ctr_drbg_context structure. - * \param output The buffer to fill. - * \param output_len The length of the buffer in bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED or - * #MBEDTLS_ERR_CTR_DRBG_REQUEST_TOO_BIG on failure. - */ -int mbedtls_ctr_drbg_random(void *p_rng, - unsigned char *output, size_t output_len); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief This function writes a seed file. - * - * \param ctx The CTR_DRBG context. - * \param path The name of the file. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_FILE_IO_ERROR on file error. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED on reseed - * failure. - */ -int mbedtls_ctr_drbg_write_seed_file(mbedtls_ctr_drbg_context *ctx, const char *path); - -/** - * \brief This function reads and updates a seed file. The seed - * is added to this instance. - * - * \param ctx The CTR_DRBG context. - * \param path The name of the file. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_CTR_DRBG_FILE_IO_ERROR on file error. - * \return #MBEDTLS_ERR_CTR_DRBG_ENTROPY_SOURCE_FAILED on - * reseed failure. - * \return #MBEDTLS_ERR_CTR_DRBG_INPUT_TOO_BIG if the existing - * seed file is too large. - */ -int mbedtls_ctr_drbg_update_seed_file(mbedtls_ctr_drbg_context *ctx, const char *path); -#endif /* MBEDTLS_FS_IO */ - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The CTR_DRBG checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_ctr_drbg_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* ctr_drbg.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/debug.h b/android/mbedtls/arm64-v8a/include/mbedtls/debug.h deleted file mode 100644 index e6f5dadb1..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/debug.h +++ /dev/null @@ -1,156 +0,0 @@ -/** - * \file debug.h - * - * \brief Functions for controlling and providing debug output from the library. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_DEBUG_H -#define MBEDTLS_DEBUG_H - -#include "mbedtls/build_info.h" - -#include "mbedtls/ssl.h" - -#if defined(MBEDTLS_ECP_C) -#include "mbedtls/ecp.h" -#endif - -#if defined(MBEDTLS_DEBUG_C) - -#define MBEDTLS_DEBUG_STRIP_PARENS(...) __VA_ARGS__ - -#define MBEDTLS_SSL_DEBUG_MSG(level, args) \ - mbedtls_debug_print_msg(ssl, level, __FILE__, __LINE__, \ - MBEDTLS_DEBUG_STRIP_PARENS args) - -#define MBEDTLS_SSL_DEBUG_RET(level, text, ret) \ - mbedtls_debug_print_ret(ssl, level, __FILE__, __LINE__, text, ret) - -#define MBEDTLS_SSL_DEBUG_BUF(level, text, buf, len) \ - mbedtls_debug_print_buf(ssl, level, __FILE__, __LINE__, text, buf, len) - -#if defined(MBEDTLS_BIGNUM_C) -#define MBEDTLS_SSL_DEBUG_MPI(level, text, X) \ - mbedtls_debug_print_mpi(ssl, level, __FILE__, __LINE__, text, X) -#endif - -#if defined(MBEDTLS_ECP_C) -#define MBEDTLS_SSL_DEBUG_ECP(level, text, X) \ - mbedtls_debug_print_ecp(ssl, level, __FILE__, __LINE__, text, X) -#endif - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -#if !defined(MBEDTLS_X509_REMOVE_INFO) -#define MBEDTLS_SSL_DEBUG_CRT(level, text, crt) \ - mbedtls_debug_print_crt(ssl, level, __FILE__, __LINE__, text, crt) -#else -#define MBEDTLS_SSL_DEBUG_CRT(level, text, crt) do { } while (0) -#endif /* MBEDTLS_X509_REMOVE_INFO */ -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_ECDH_C) -#define MBEDTLS_SSL_DEBUG_ECDH(level, ecdh, attr) \ - mbedtls_debug_printf_ecdh(ssl, level, __FILE__, __LINE__, ecdh, attr) -#endif - -#else /* MBEDTLS_DEBUG_C */ - -#define MBEDTLS_SSL_DEBUG_MSG(level, args) do { } while (0) -#define MBEDTLS_SSL_DEBUG_RET(level, text, ret) do { } while (0) -#define MBEDTLS_SSL_DEBUG_BUF(level, text, buf, len) do { } while (0) -#define MBEDTLS_SSL_DEBUG_MPI(level, text, X) do { } while (0) -#define MBEDTLS_SSL_DEBUG_ECP(level, text, X) do { } while (0) -#define MBEDTLS_SSL_DEBUG_CRT(level, text, crt) do { } while (0) -#define MBEDTLS_SSL_DEBUG_ECDH(level, ecdh, attr) do { } while (0) - -#endif /* MBEDTLS_DEBUG_C */ - -/** - * \def MBEDTLS_PRINTF_ATTRIBUTE - * - * Mark a function as having printf attributes, and thus enable checking - * via -wFormat and other flags. This does nothing on builds with compilers - * that do not support the format attribute - * - * Module: library/debug.c - * Caller: - * - * This module provides debugging functions. - */ -#if defined(__has_attribute) -#if __has_attribute(format) -#if defined(__MINGW32__) && __USE_MINGW_ANSI_STDIO == 1 -#define MBEDTLS_PRINTF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(gnu_printf, string_index, first_to_check))) -#else /* defined(__MINGW32__) && __USE_MINGW_ANSI_STDIO == 1 */ -#define MBEDTLS_PRINTF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((format(printf, string_index, first_to_check))) -#endif -#else /* __has_attribute(format) */ -#define MBEDTLS_PRINTF_ATTRIBUTE(string_index, first_to_check) -#endif /* __has_attribute(format) */ -#else /* defined(__has_attribute) */ -#define MBEDTLS_PRINTF_ATTRIBUTE(string_index, first_to_check) -#endif - -/** - * \def MBEDTLS_PRINTF_SIZET - * - * MBEDTLS_PRINTF_xxx: Due to issues with older window compilers - * and MinGW we need to define the printf specifier for size_t - * and long long per platform. - * - * Module: library/debug.c - * Caller: - * - * This module provides debugging functions. - */ -#if defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER < 1900) - #include - #define MBEDTLS_PRINTF_SIZET PRIuPTR - #define MBEDTLS_PRINTF_LONGLONG "I64d" -#else \ - /* defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER < 1900) */ - #define MBEDTLS_PRINTF_SIZET "zu" - #define MBEDTLS_PRINTF_LONGLONG "lld" -#endif \ - /* defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER < 1900) */ - -#if !defined(MBEDTLS_PRINTF_MS_TIME) -#include -#if !defined(PRId64) -#define MBEDTLS_PRINTF_MS_TIME MBEDTLS_PRINTF_LONGLONG -#else -#define MBEDTLS_PRINTF_MS_TIME PRId64 -#endif -#endif /* MBEDTLS_PRINTF_MS_TIME */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Set the threshold error level to handle globally all debug output. - * Debug messages that have a level over the threshold value are - * discarded. - * (Default value: 0 = No debug ) - * - * \param threshold threshold level of messages to filter on. Messages at a - * higher level will be discarded. - * - Debug levels - * - 0 No debug - * - 1 Error - * - 2 State change - * - 3 Informational - * - 4 Verbose - */ -void mbedtls_debug_set_threshold(int threshold); - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_DEBUG_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/des.h b/android/mbedtls/arm64-v8a/include/mbedtls/des.h deleted file mode 100644 index 2b097a13d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/des.h +++ /dev/null @@ -1,385 +0,0 @@ -/** - * \file des.h - * - * \brief DES block cipher - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - * - */ -#ifndef MBEDTLS_DES_H -#define MBEDTLS_DES_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include -#include - -#define MBEDTLS_DES_ENCRYPT 1 -#define MBEDTLS_DES_DECRYPT 0 - -/** The data input has an invalid length. */ -#define MBEDTLS_ERR_DES_INVALID_INPUT_LENGTH -0x0032 - -#define MBEDTLS_DES_KEY_SIZE 8 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_DES_ALT) -// Regular implementation -// - -/** - * \brief DES context structure - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -typedef struct mbedtls_des_context { - uint32_t MBEDTLS_PRIVATE(sk)[32]; /*!< DES subkeys */ -} -mbedtls_des_context; - -/** - * \brief Triple-DES context structure - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -typedef struct mbedtls_des3_context { - uint32_t MBEDTLS_PRIVATE(sk)[96]; /*!< 3DES subkeys */ -} -mbedtls_des3_context; - -#else /* MBEDTLS_DES_ALT */ -#include "des_alt.h" -#endif /* MBEDTLS_DES_ALT */ - -/** - * \brief Initialize DES context - * - * \param ctx DES context to be initialized - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des_init(mbedtls_des_context *ctx); - -/** - * \brief Clear DES context - * - * \param ctx DES context to be cleared - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des_free(mbedtls_des_context *ctx); - -/** - * \brief Initialize Triple-DES context - * - * \param ctx DES3 context to be initialized - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des3_init(mbedtls_des3_context *ctx); - -/** - * \brief Clear Triple-DES context - * - * \param ctx DES3 context to be cleared - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des3_free(mbedtls_des3_context *ctx); - -/** - * \brief Set key parity on the given key to odd. - * - * DES keys are 56 bits long, but each byte is padded with - * a parity bit to allow verification. - * - * \param key 8-byte secret key - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des_key_set_parity(unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -/** - * \brief Check that key parity on the given key is odd. - * - * DES keys are 56 bits long, but each byte is padded with - * a parity bit to allow verification. - * - * \param key 8-byte secret key - * - * \return 0 is parity was ok, 1 if parity was not correct. - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_key_check_key_parity(const unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -/** - * \brief Check that key is not a weak or semi-weak DES key - * - * \param key 8-byte secret key - * - * \return 0 if no weak key was found, 1 if a weak key was identified. - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_key_check_weak(const unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -/** - * \brief DES key schedule (56-bit, encryption) - * - * \param ctx DES context to be initialized - * \param key 8-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_setkey_enc(mbedtls_des_context *ctx, const unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -/** - * \brief DES key schedule (56-bit, decryption) - * - * \param ctx DES context to be initialized - * \param key 8-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_setkey_dec(mbedtls_des_context *ctx, const unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -/** - * \brief Triple-DES key schedule (112-bit, encryption) - * - * \param ctx 3DES context to be initialized - * \param key 16-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_set2key_enc(mbedtls_des3_context *ctx, - const unsigned char key[MBEDTLS_DES_KEY_SIZE * 2]); - -/** - * \brief Triple-DES key schedule (112-bit, decryption) - * - * \param ctx 3DES context to be initialized - * \param key 16-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_set2key_dec(mbedtls_des3_context *ctx, - const unsigned char key[MBEDTLS_DES_KEY_SIZE * 2]); - -/** - * \brief Triple-DES key schedule (168-bit, encryption) - * - * \param ctx 3DES context to be initialized - * \param key 24-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_set3key_enc(mbedtls_des3_context *ctx, - const unsigned char key[MBEDTLS_DES_KEY_SIZE * 3]); - -/** - * \brief Triple-DES key schedule (168-bit, decryption) - * - * \param ctx 3DES context to be initialized - * \param key 24-byte secret key - * - * \return 0 - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_set3key_dec(mbedtls_des3_context *ctx, - const unsigned char key[MBEDTLS_DES_KEY_SIZE * 3]); - -/** - * \brief DES-ECB block encryption/decryption - * - * \param ctx DES context - * \param input 64-bit input block - * \param output 64-bit output block - * - * \return 0 if successful - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_crypt_ecb(mbedtls_des_context *ctx, - const unsigned char input[8], - unsigned char output[8]); - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -/** - * \brief DES-CBC buffer encryption/decryption - * - * \note Upon exit, the content of the IV is updated so that you can - * call the function same function again on the following - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If on the other hand you need to retain the contents of the - * IV, you should either save it manually or use the cipher - * module instead. - * - * \param ctx DES context - * \param mode MBEDTLS_DES_ENCRYPT or MBEDTLS_DES_DECRYPT - * \param length length of the input data - * \param iv initialization vector (updated after use) - * \param input buffer holding the input data - * \param output buffer holding the output data - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des_crypt_cbc(mbedtls_des_context *ctx, - int mode, - size_t length, - unsigned char iv[8], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CBC */ - -/** - * \brief 3DES-ECB block encryption/decryption - * - * \param ctx 3DES context - * \param input 64-bit input block - * \param output 64-bit output block - * - * \return 0 if successful - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_crypt_ecb(mbedtls_des3_context *ctx, - const unsigned char input[8], - unsigned char output[8]); - -#if defined(MBEDTLS_CIPHER_MODE_CBC) -/** - * \brief 3DES-CBC buffer encryption/decryption - * - * \note Upon exit, the content of the IV is updated so that you can - * call the function same function again on the following - * block(s) of data and get the same result as if it was - * encrypted in one call. This allows a "streaming" usage. - * If on the other hand you need to retain the contents of the - * IV, you should either save it manually or use the cipher - * module instead. - * - * \param ctx 3DES context - * \param mode MBEDTLS_DES_ENCRYPT or MBEDTLS_DES_DECRYPT - * \param length length of the input data - * \param iv initialization vector (updated after use) - * \param input buffer holding the input data - * \param output buffer holding the output data - * - * \return 0 if successful, or MBEDTLS_ERR_DES_INVALID_INPUT_LENGTH - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_des3_crypt_cbc(mbedtls_des3_context *ctx, - int mode, - size_t length, - unsigned char iv[8], - const unsigned char *input, - unsigned char *output); -#endif /* MBEDTLS_CIPHER_MODE_CBC */ - -/** - * \brief Internal function for key expansion. - * (Only exposed to allow overriding it, - * see MBEDTLS_DES_SETKEY_ALT) - * - * \param SK Round keys - * \param key Base key - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers - * instead. - */ -void mbedtls_des_setkey(uint32_t SK[32], - const unsigned char key[MBEDTLS_DES_KEY_SIZE]); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -MBEDTLS_CHECK_RETURN_CRITICAL -int mbedtls_des_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* des.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/dhm.h b/android/mbedtls/arm64-v8a/include/mbedtls/dhm.h deleted file mode 100644 index bbfe6ea8d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/dhm.h +++ /dev/null @@ -1,972 +0,0 @@ -/** - * \file dhm.h - * - * \brief This file contains Diffie-Hellman-Merkle (DHM) key exchange - * definitions and functions. - * - * Diffie-Hellman-Merkle (DHM) key exchange is defined in - * RFC-2631: Diffie-Hellman Key Agreement Method and - * Public-Key Cryptography Standards (PKCS) #3: Diffie - * Hellman Key Agreement Standard. - * - * RFC-3526: More Modular Exponential (MODP) Diffie-Hellman groups for - * Internet Key Exchange (IKE) defines a number of standardized - * Diffie-Hellman groups for IKE. - * - * RFC-5114: Additional Diffie-Hellman Groups for Use with IETF - * Standards defines a number of standardized Diffie-Hellman - * groups that can be used. - * - * \warning The security of the DHM key exchange relies on the proper choice - * of prime modulus - optimally, it should be a safe prime. The usage - * of non-safe primes both decreases the difficulty of the underlying - * discrete logarithm problem and can lead to small subgroup attacks - * leaking private exponent bits when invalid public keys are used - * and not detected. This is especially relevant if the same DHM - * parameters are reused for multiple key exchanges as in static DHM, - * while the criticality of small-subgroup attacks is lower for - * ephemeral DHM. - * - * \warning For performance reasons, the code does neither perform primality - * nor safe primality tests, nor the expensive checks for invalid - * subgroups. Moreover, even if these were performed, non-standardized - * primes cannot be trusted because of the possibility of backdoors - * that can't be effectively checked for. - * - * \warning Diffie-Hellman-Merkle is therefore a security risk when not using - * standardized primes generated using a trustworthy ("nothing up - * my sleeve") method, such as the RFC 3526 / 7919 primes. In the TLS - * protocol, DH parameters need to be negotiated, so using the default - * primes systematically is not always an option. If possible, use - * Elliptic Curve Diffie-Hellman (ECDH), which has better performance, - * and for which the TLS protocol mandates the use of standard - * parameters. - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_DHM_H -#define MBEDTLS_DHM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/bignum.h" - -/* - * DHM Error codes - */ -/** Bad input parameters. */ -#define MBEDTLS_ERR_DHM_BAD_INPUT_DATA -0x3080 -/** Reading of the DHM parameters failed. */ -#define MBEDTLS_ERR_DHM_READ_PARAMS_FAILED -0x3100 -/** Making of the DHM parameters failed. */ -#define MBEDTLS_ERR_DHM_MAKE_PARAMS_FAILED -0x3180 -/** Reading of the public values failed. */ -#define MBEDTLS_ERR_DHM_READ_PUBLIC_FAILED -0x3200 -/** Making of the public value failed. */ -#define MBEDTLS_ERR_DHM_MAKE_PUBLIC_FAILED -0x3280 -/** Calculation of the DHM secret failed. */ -#define MBEDTLS_ERR_DHM_CALC_SECRET_FAILED -0x3300 -/** The ASN.1 data is not formatted correctly. */ -#define MBEDTLS_ERR_DHM_INVALID_FORMAT -0x3380 -/** Allocation of memory failed. */ -#define MBEDTLS_ERR_DHM_ALLOC_FAILED -0x3400 -/** Read or write of file failed. */ -#define MBEDTLS_ERR_DHM_FILE_IO_ERROR -0x3480 -/** Setting the modulus and generator failed. */ -#define MBEDTLS_ERR_DHM_SET_GROUP_FAILED -0x3580 - -/** Which parameter to access in mbedtls_dhm_get_value(). */ -typedef enum { - MBEDTLS_DHM_PARAM_P, /*!< The prime modulus. */ - MBEDTLS_DHM_PARAM_G, /*!< The generator. */ - MBEDTLS_DHM_PARAM_X, /*!< Our secret value. */ - MBEDTLS_DHM_PARAM_GX, /*!< Our public key = \c G^X mod \c P. */ - MBEDTLS_DHM_PARAM_GY, /*!< The public key of the peer = \c G^Y mod \c P. */ - MBEDTLS_DHM_PARAM_K, /*!< The shared secret = \c G^(XY) mod \c P. */ -} mbedtls_dhm_parameter; - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_DHM_ALT) - -/** - * \brief The DHM context structure. - */ -typedef struct mbedtls_dhm_context { - mbedtls_mpi MBEDTLS_PRIVATE(P); /*!< The prime modulus. */ - mbedtls_mpi MBEDTLS_PRIVATE(G); /*!< The generator. */ - mbedtls_mpi MBEDTLS_PRIVATE(X); /*!< Our secret value. */ - mbedtls_mpi MBEDTLS_PRIVATE(GX); /*!< Our public key = \c G^X mod \c P. */ - mbedtls_mpi MBEDTLS_PRIVATE(GY); /*!< The public key of the peer = \c G^Y mod \c P. */ - mbedtls_mpi MBEDTLS_PRIVATE(K); /*!< The shared secret = \c G^(XY) mod \c P. */ - mbedtls_mpi MBEDTLS_PRIVATE(RP); /*!< The cached value = \c R^2 mod \c P. */ - mbedtls_mpi MBEDTLS_PRIVATE(Vi); /*!< The blinding value. */ - mbedtls_mpi MBEDTLS_PRIVATE(Vf); /*!< The unblinding value. */ - mbedtls_mpi MBEDTLS_PRIVATE(pX); /*!< The previous \c X. */ -} -mbedtls_dhm_context; - -#else /* MBEDTLS_DHM_ALT */ -#include "dhm_alt.h" -#endif /* MBEDTLS_DHM_ALT */ - -/** - * \brief This function initializes the DHM context. - * - * \param ctx The DHM context to initialize. - */ -void mbedtls_dhm_init(mbedtls_dhm_context *ctx); - -/** - * \brief This function parses the DHM parameters in a - * TLS ServerKeyExchange handshake message - * (DHM modulus, generator, and public key). - * - * \note In a TLS handshake, this is the how the client - * sets up its DHM context from the server's public - * DHM key material. - * - * \param ctx The DHM context to use. This must be initialized. - * \param p On input, *p must be the start of the input buffer. - * On output, *p is updated to point to the end of the data - * that has been read. On success, this is the first byte - * past the end of the ServerKeyExchange parameters. - * On error, this is the point at which an error has been - * detected, which is usually not useful except to debug - * failures. - * \param end The end of the input buffer. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_read_params(mbedtls_dhm_context *ctx, - unsigned char **p, - const unsigned char *end); - -/** - * \brief This function generates a DHM key pair and exports its - * public part together with the DHM parameters in the format - * used in a TLS ServerKeyExchange handshake message. - * - * \note This function assumes that the DHM parameters \c ctx->P - * and \c ctx->G have already been properly set. For that, use - * mbedtls_dhm_set_group() below in conjunction with - * mbedtls_mpi_read_binary() and mbedtls_mpi_read_string(). - * - * \note In a TLS handshake, this is the how the server generates - * and exports its DHM key material. - * - * \param ctx The DHM context to use. This must be initialized - * and have the DHM parameters set. It may or may not - * already have imported the peer's public key. - * \param x_size The private key size in Bytes. - * \param olen The address at which to store the number of Bytes - * written on success. This must not be \c NULL. - * \param output The destination buffer. This must be a writable buffer of - * sufficient size to hold the reduced binary presentation of - * the modulus, the generator and the public key, each wrapped - * with a 2-byte length field. It is the responsibility of the - * caller to ensure that enough space is available. Refer to - * mbedtls_mpi_size() to computing the byte-size of an MPI. - * \param f_rng The RNG function. Must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context parameter. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_make_params(mbedtls_dhm_context *ctx, int x_size, - unsigned char *output, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function sets the prime modulus and generator. - * - * \note This function can be used to set \c ctx->P, \c ctx->G - * in preparation for mbedtls_dhm_make_params(). - * - * \param ctx The DHM context to configure. This must be initialized. - * \param P The MPI holding the DHM prime modulus. This must be - * an initialized MPI. - * \param G The MPI holding the DHM generator. This must be an - * initialized MPI. - * - * \return \c 0 if successful. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_set_group(mbedtls_dhm_context *ctx, - const mbedtls_mpi *P, - const mbedtls_mpi *G); - -/** - * \brief This function imports the raw public value of the peer. - * - * \note In a TLS handshake, this is the how the server imports - * the Client's public DHM key. - * - * \param ctx The DHM context to use. This must be initialized and have - * its DHM parameters set, e.g. via mbedtls_dhm_set_group(). - * It may or may not already have generated its own private key. - * \param input The input buffer containing the \c G^Y value of the peer. - * This must be a readable buffer of size \p ilen Bytes. - * \param ilen The size of the input buffer \p input in Bytes. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_read_public(mbedtls_dhm_context *ctx, - const unsigned char *input, size_t ilen); - -/** - * \brief This function creates a DHM key pair and exports - * the raw public key in big-endian format. - * - * \note The destination buffer is always fully written - * so as to contain a big-endian representation of G^X mod P. - * If it is larger than \c ctx->len, it is padded accordingly - * with zero-bytes at the beginning. - * - * \param ctx The DHM context to use. This must be initialized and - * have the DHM parameters set. It may or may not already - * have imported the peer's public key. - * \param x_size The private key size in Bytes. - * \param output The destination buffer. This must be a writable buffer of - * size \p olen Bytes. - * \param olen The length of the destination buffer. This must be at least - * equal to `ctx->len` (the size of \c P). - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_make_public(mbedtls_dhm_context *ctx, int x_size, - unsigned char *output, size_t olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function derives and exports the shared secret - * \c (G^Y)^X mod \c P. - * - * \note If \p f_rng is not \c NULL, it is used to blind the input as - * a countermeasure against timing attacks. Blinding is used - * only if our private key \c X is re-used, and not used - * otherwise. We recommend always passing a non-NULL - * \p f_rng argument. - * - * \param ctx The DHM context to use. This must be initialized - * and have its own private key generated and the peer's - * public key imported. - * \param output The buffer to write the generated shared key to. This - * must be a writable buffer of size \p output_size Bytes. - * \param output_size The size of the destination buffer. This must be at - * least the size of \c ctx->len (the size of \c P). - * \param olen On exit, holds the actual number of Bytes written. - * \param f_rng The RNG function. Must not be \c NULL. Used for - * blinding. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context parameter. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX error code on failure. - */ -int mbedtls_dhm_calc_secret(mbedtls_dhm_context *ctx, - unsigned char *output, size_t output_size, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function returns the size of the prime modulus in bits. - * - * \param ctx The DHM context to query. - * - * \return The size of the prime modulus in bits, - * i.e. the number n such that 2^(n-1) <= P < 2^n. - */ -size_t mbedtls_dhm_get_bitlen(const mbedtls_dhm_context *ctx); - -/** - * \brief This function returns the size of the prime modulus in bytes. - * - * \param ctx The DHM context to query. - * - * \return The size of the prime modulus in bytes, - * i.e. the number n such that 2^(8*(n-1)) <= P < 2^(8*n). - */ -size_t mbedtls_dhm_get_len(const mbedtls_dhm_context *ctx); - -/** - * \brief This function copies a parameter of a DHM key. - * - * \param ctx The DHM context to query. - * \param param The parameter to copy. - * \param dest The MPI object to copy the value into. It must be - * initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_DHM_BAD_INPUT_DATA if \p param is invalid. - * \return An \c MBEDTLS_ERR_MPI_XXX error code if the copy fails. - */ -int mbedtls_dhm_get_value(const mbedtls_dhm_context *ctx, - mbedtls_dhm_parameter param, - mbedtls_mpi *dest); - -/** - * \brief This function frees and clears the components - * of a DHM context. - * - * \param ctx The DHM context to free and clear. This may be \c NULL, - * in which case this function is a no-op. If it is not \c NULL, - * it must point to an initialized DHM context. - */ -void mbedtls_dhm_free(mbedtls_dhm_context *ctx); - -#if defined(MBEDTLS_ASN1_PARSE_C) -/** - * \brief This function parses DHM parameters in PEM or DER format. - * - * \param dhm The DHM context to import the DHM parameters into. - * This must be initialized. - * \param dhmin The input buffer. This must be a readable buffer of - * length \p dhminlen Bytes. - * \param dhminlen The size of the input buffer \p dhmin, including the - * terminating \c NULL Byte for PEM data. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX or \c MBEDTLS_ERR_PEM_XXX error - * code on failure. - */ -int mbedtls_dhm_parse_dhm(mbedtls_dhm_context *dhm, const unsigned char *dhmin, - size_t dhminlen); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief This function loads and parses DHM parameters from a file. - * - * \param dhm The DHM context to load the parameters to. - * This must be initialized. - * \param path The filename to read the DHM parameters from. - * This must not be \c NULL. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_DHM_XXX or \c MBEDTLS_ERR_PEM_XXX - * error code on failure. - */ -int mbedtls_dhm_parse_dhmfile(mbedtls_dhm_context *dhm, const char *path); -#endif /* MBEDTLS_FS_IO */ -#endif /* MBEDTLS_ASN1_PARSE_C */ - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The DMH checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_dhm_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ -#ifdef __cplusplus -} -#endif - -/** - * RFC 3526, RFC 5114 and RFC 7919 standardize a number of - * Diffie-Hellman groups, some of which are included here - * for use within the SSL/TLS module and the user's convenience - * when configuring the Diffie-Hellman parameters by hand - * through \c mbedtls_ssl_conf_dh_param. - * - * The following lists the source of the above groups in the standards: - * - RFC 5114 section 2.2: 2048-bit MODP Group with 224-bit Prime Order Subgroup - * - RFC 3526 section 3: 2048-bit MODP Group - * - RFC 3526 section 4: 3072-bit MODP Group - * - RFC 3526 section 5: 4096-bit MODP Group - * - RFC 7919 section A.1: ffdhe2048 - * - RFC 7919 section A.2: ffdhe3072 - * - RFC 7919 section A.3: ffdhe4096 - * - RFC 7919 section A.4: ffdhe6144 - * - RFC 7919 section A.5: ffdhe8192 - * - * The constants with suffix "_p" denote the chosen prime moduli, while - * the constants with suffix "_g" denote the chosen generator - * of the associated prime field. - * - * The constants further suffixed with "_bin" are provided in binary format, - * while all other constants represent null-terminated strings holding the - * hexadecimal presentation of the respective numbers. - * - * The primes from RFC 3526 and RFC 7919 have been generating by the following - * trust-worthy procedure: - * - Fix N in { 2048, 3072, 4096, 6144, 8192 } and consider the N-bit number - * the first and last 64 bits are all 1, and the remaining N - 128 bits of - * which are 0x7ff...ff. - * - Add the smallest multiple of the first N - 129 bits of the binary expansion - * of pi (for RFC 5236) or e (for RFC 7919) to this intermediate bit-string - * such that the resulting integer is a safe-prime. - * - The result is the respective RFC 3526 / 7919 prime, and the corresponding - * generator is always chosen to be 2 (which is a square for these prime, - * hence the corresponding subgroup has order (p-1)/2 and avoids leaking a - * bit in the private exponent). - * - */ - -/* - * Trustworthy DHM parameters in binary form - */ - -#define MBEDTLS_DHM_RFC3526_MODP_2048_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xC9, 0x0F, 0xDA, 0xA2, 0x21, 0x68, 0xC2, 0x34, \ - 0xC4, 0xC6, 0x62, 0x8B, 0x80, 0xDC, 0x1C, 0xD1, \ - 0x29, 0x02, 0x4E, 0x08, 0x8A, 0x67, 0xCC, 0x74, \ - 0x02, 0x0B, 0xBE, 0xA6, 0x3B, 0x13, 0x9B, 0x22, \ - 0x51, 0x4A, 0x08, 0x79, 0x8E, 0x34, 0x04, 0xDD, \ - 0xEF, 0x95, 0x19, 0xB3, 0xCD, 0x3A, 0x43, 0x1B, \ - 0x30, 0x2B, 0x0A, 0x6D, 0xF2, 0x5F, 0x14, 0x37, \ - 0x4F, 0xE1, 0x35, 0x6D, 0x6D, 0x51, 0xC2, 0x45, \ - 0xE4, 0x85, 0xB5, 0x76, 0x62, 0x5E, 0x7E, 0xC6, \ - 0xF4, 0x4C, 0x42, 0xE9, 0xA6, 0x37, 0xED, 0x6B, \ - 0x0B, 0xFF, 0x5C, 0xB6, 0xF4, 0x06, 0xB7, 0xED, \ - 0xEE, 0x38, 0x6B, 0xFB, 0x5A, 0x89, 0x9F, 0xA5, \ - 0xAE, 0x9F, 0x24, 0x11, 0x7C, 0x4B, 0x1F, 0xE6, \ - 0x49, 0x28, 0x66, 0x51, 0xEC, 0xE4, 0x5B, 0x3D, \ - 0xC2, 0x00, 0x7C, 0xB8, 0xA1, 0x63, 0xBF, 0x05, \ - 0x98, 0xDA, 0x48, 0x36, 0x1C, 0x55, 0xD3, 0x9A, \ - 0x69, 0x16, 0x3F, 0xA8, 0xFD, 0x24, 0xCF, 0x5F, \ - 0x83, 0x65, 0x5D, 0x23, 0xDC, 0xA3, 0xAD, 0x96, \ - 0x1C, 0x62, 0xF3, 0x56, 0x20, 0x85, 0x52, 0xBB, \ - 0x9E, 0xD5, 0x29, 0x07, 0x70, 0x96, 0x96, 0x6D, \ - 0x67, 0x0C, 0x35, 0x4E, 0x4A, 0xBC, 0x98, 0x04, \ - 0xF1, 0x74, 0x6C, 0x08, 0xCA, 0x18, 0x21, 0x7C, \ - 0x32, 0x90, 0x5E, 0x46, 0x2E, 0x36, 0xCE, 0x3B, \ - 0xE3, 0x9E, 0x77, 0x2C, 0x18, 0x0E, 0x86, 0x03, \ - 0x9B, 0x27, 0x83, 0xA2, 0xEC, 0x07, 0xA2, 0x8F, \ - 0xB5, 0xC5, 0x5D, 0xF0, 0x6F, 0x4C, 0x52, 0xC9, \ - 0xDE, 0x2B, 0xCB, 0xF6, 0x95, 0x58, 0x17, 0x18, \ - 0x39, 0x95, 0x49, 0x7C, 0xEA, 0x95, 0x6A, 0xE5, \ - 0x15, 0xD2, 0x26, 0x18, 0x98, 0xFA, 0x05, 0x10, \ - 0x15, 0x72, 0x8E, 0x5A, 0x8A, 0xAC, 0xAA, 0x68, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC3526_MODP_2048_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC3526_MODP_3072_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xC9, 0x0F, 0xDA, 0xA2, 0x21, 0x68, 0xC2, 0x34, \ - 0xC4, 0xC6, 0x62, 0x8B, 0x80, 0xDC, 0x1C, 0xD1, \ - 0x29, 0x02, 0x4E, 0x08, 0x8A, 0x67, 0xCC, 0x74, \ - 0x02, 0x0B, 0xBE, 0xA6, 0x3B, 0x13, 0x9B, 0x22, \ - 0x51, 0x4A, 0x08, 0x79, 0x8E, 0x34, 0x04, 0xDD, \ - 0xEF, 0x95, 0x19, 0xB3, 0xCD, 0x3A, 0x43, 0x1B, \ - 0x30, 0x2B, 0x0A, 0x6D, 0xF2, 0x5F, 0x14, 0x37, \ - 0x4F, 0xE1, 0x35, 0x6D, 0x6D, 0x51, 0xC2, 0x45, \ - 0xE4, 0x85, 0xB5, 0x76, 0x62, 0x5E, 0x7E, 0xC6, \ - 0xF4, 0x4C, 0x42, 0xE9, 0xA6, 0x37, 0xED, 0x6B, \ - 0x0B, 0xFF, 0x5C, 0xB6, 0xF4, 0x06, 0xB7, 0xED, \ - 0xEE, 0x38, 0x6B, 0xFB, 0x5A, 0x89, 0x9F, 0xA5, \ - 0xAE, 0x9F, 0x24, 0x11, 0x7C, 0x4B, 0x1F, 0xE6, \ - 0x49, 0x28, 0x66, 0x51, 0xEC, 0xE4, 0x5B, 0x3D, \ - 0xC2, 0x00, 0x7C, 0xB8, 0xA1, 0x63, 0xBF, 0x05, \ - 0x98, 0xDA, 0x48, 0x36, 0x1C, 0x55, 0xD3, 0x9A, \ - 0x69, 0x16, 0x3F, 0xA8, 0xFD, 0x24, 0xCF, 0x5F, \ - 0x83, 0x65, 0x5D, 0x23, 0xDC, 0xA3, 0xAD, 0x96, \ - 0x1C, 0x62, 0xF3, 0x56, 0x20, 0x85, 0x52, 0xBB, \ - 0x9E, 0xD5, 0x29, 0x07, 0x70, 0x96, 0x96, 0x6D, \ - 0x67, 0x0C, 0x35, 0x4E, 0x4A, 0xBC, 0x98, 0x04, \ - 0xF1, 0x74, 0x6C, 0x08, 0xCA, 0x18, 0x21, 0x7C, \ - 0x32, 0x90, 0x5E, 0x46, 0x2E, 0x36, 0xCE, 0x3B, \ - 0xE3, 0x9E, 0x77, 0x2C, 0x18, 0x0E, 0x86, 0x03, \ - 0x9B, 0x27, 0x83, 0xA2, 0xEC, 0x07, 0xA2, 0x8F, \ - 0xB5, 0xC5, 0x5D, 0xF0, 0x6F, 0x4C, 0x52, 0xC9, \ - 0xDE, 0x2B, 0xCB, 0xF6, 0x95, 0x58, 0x17, 0x18, \ - 0x39, 0x95, 0x49, 0x7C, 0xEA, 0x95, 0x6A, 0xE5, \ - 0x15, 0xD2, 0x26, 0x18, 0x98, 0xFA, 0x05, 0x10, \ - 0x15, 0x72, 0x8E, 0x5A, 0x8A, 0xAA, 0xC4, 0x2D, \ - 0xAD, 0x33, 0x17, 0x0D, 0x04, 0x50, 0x7A, 0x33, \ - 0xA8, 0x55, 0x21, 0xAB, 0xDF, 0x1C, 0xBA, 0x64, \ - 0xEC, 0xFB, 0x85, 0x04, 0x58, 0xDB, 0xEF, 0x0A, \ - 0x8A, 0xEA, 0x71, 0x57, 0x5D, 0x06, 0x0C, 0x7D, \ - 0xB3, 0x97, 0x0F, 0x85, 0xA6, 0xE1, 0xE4, 0xC7, \ - 0xAB, 0xF5, 0xAE, 0x8C, 0xDB, 0x09, 0x33, 0xD7, \ - 0x1E, 0x8C, 0x94, 0xE0, 0x4A, 0x25, 0x61, 0x9D, \ - 0xCE, 0xE3, 0xD2, 0x26, 0x1A, 0xD2, 0xEE, 0x6B, \ - 0xF1, 0x2F, 0xFA, 0x06, 0xD9, 0x8A, 0x08, 0x64, \ - 0xD8, 0x76, 0x02, 0x73, 0x3E, 0xC8, 0x6A, 0x64, \ - 0x52, 0x1F, 0x2B, 0x18, 0x17, 0x7B, 0x20, 0x0C, \ - 0xBB, 0xE1, 0x17, 0x57, 0x7A, 0x61, 0x5D, 0x6C, \ - 0x77, 0x09, 0x88, 0xC0, 0xBA, 0xD9, 0x46, 0xE2, \ - 0x08, 0xE2, 0x4F, 0xA0, 0x74, 0xE5, 0xAB, 0x31, \ - 0x43, 0xDB, 0x5B, 0xFC, 0xE0, 0xFD, 0x10, 0x8E, \ - 0x4B, 0x82, 0xD1, 0x20, 0xA9, 0x3A, 0xD2, 0xCA, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC3526_MODP_3072_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC3526_MODP_4096_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xC9, 0x0F, 0xDA, 0xA2, 0x21, 0x68, 0xC2, 0x34, \ - 0xC4, 0xC6, 0x62, 0x8B, 0x80, 0xDC, 0x1C, 0xD1, \ - 0x29, 0x02, 0x4E, 0x08, 0x8A, 0x67, 0xCC, 0x74, \ - 0x02, 0x0B, 0xBE, 0xA6, 0x3B, 0x13, 0x9B, 0x22, \ - 0x51, 0x4A, 0x08, 0x79, 0x8E, 0x34, 0x04, 0xDD, \ - 0xEF, 0x95, 0x19, 0xB3, 0xCD, 0x3A, 0x43, 0x1B, \ - 0x30, 0x2B, 0x0A, 0x6D, 0xF2, 0x5F, 0x14, 0x37, \ - 0x4F, 0xE1, 0x35, 0x6D, 0x6D, 0x51, 0xC2, 0x45, \ - 0xE4, 0x85, 0xB5, 0x76, 0x62, 0x5E, 0x7E, 0xC6, \ - 0xF4, 0x4C, 0x42, 0xE9, 0xA6, 0x37, 0xED, 0x6B, \ - 0x0B, 0xFF, 0x5C, 0xB6, 0xF4, 0x06, 0xB7, 0xED, \ - 0xEE, 0x38, 0x6B, 0xFB, 0x5A, 0x89, 0x9F, 0xA5, \ - 0xAE, 0x9F, 0x24, 0x11, 0x7C, 0x4B, 0x1F, 0xE6, \ - 0x49, 0x28, 0x66, 0x51, 0xEC, 0xE4, 0x5B, 0x3D, \ - 0xC2, 0x00, 0x7C, 0xB8, 0xA1, 0x63, 0xBF, 0x05, \ - 0x98, 0xDA, 0x48, 0x36, 0x1C, 0x55, 0xD3, 0x9A, \ - 0x69, 0x16, 0x3F, 0xA8, 0xFD, 0x24, 0xCF, 0x5F, \ - 0x83, 0x65, 0x5D, 0x23, 0xDC, 0xA3, 0xAD, 0x96, \ - 0x1C, 0x62, 0xF3, 0x56, 0x20, 0x85, 0x52, 0xBB, \ - 0x9E, 0xD5, 0x29, 0x07, 0x70, 0x96, 0x96, 0x6D, \ - 0x67, 0x0C, 0x35, 0x4E, 0x4A, 0xBC, 0x98, 0x04, \ - 0xF1, 0x74, 0x6C, 0x08, 0xCA, 0x18, 0x21, 0x7C, \ - 0x32, 0x90, 0x5E, 0x46, 0x2E, 0x36, 0xCE, 0x3B, \ - 0xE3, 0x9E, 0x77, 0x2C, 0x18, 0x0E, 0x86, 0x03, \ - 0x9B, 0x27, 0x83, 0xA2, 0xEC, 0x07, 0xA2, 0x8F, \ - 0xB5, 0xC5, 0x5D, 0xF0, 0x6F, 0x4C, 0x52, 0xC9, \ - 0xDE, 0x2B, 0xCB, 0xF6, 0x95, 0x58, 0x17, 0x18, \ - 0x39, 0x95, 0x49, 0x7C, 0xEA, 0x95, 0x6A, 0xE5, \ - 0x15, 0xD2, 0x26, 0x18, 0x98, 0xFA, 0x05, 0x10, \ - 0x15, 0x72, 0x8E, 0x5A, 0x8A, 0xAA, 0xC4, 0x2D, \ - 0xAD, 0x33, 0x17, 0x0D, 0x04, 0x50, 0x7A, 0x33, \ - 0xA8, 0x55, 0x21, 0xAB, 0xDF, 0x1C, 0xBA, 0x64, \ - 0xEC, 0xFB, 0x85, 0x04, 0x58, 0xDB, 0xEF, 0x0A, \ - 0x8A, 0xEA, 0x71, 0x57, 0x5D, 0x06, 0x0C, 0x7D, \ - 0xB3, 0x97, 0x0F, 0x85, 0xA6, 0xE1, 0xE4, 0xC7, \ - 0xAB, 0xF5, 0xAE, 0x8C, 0xDB, 0x09, 0x33, 0xD7, \ - 0x1E, 0x8C, 0x94, 0xE0, 0x4A, 0x25, 0x61, 0x9D, \ - 0xCE, 0xE3, 0xD2, 0x26, 0x1A, 0xD2, 0xEE, 0x6B, \ - 0xF1, 0x2F, 0xFA, 0x06, 0xD9, 0x8A, 0x08, 0x64, \ - 0xD8, 0x76, 0x02, 0x73, 0x3E, 0xC8, 0x6A, 0x64, \ - 0x52, 0x1F, 0x2B, 0x18, 0x17, 0x7B, 0x20, 0x0C, \ - 0xBB, 0xE1, 0x17, 0x57, 0x7A, 0x61, 0x5D, 0x6C, \ - 0x77, 0x09, 0x88, 0xC0, 0xBA, 0xD9, 0x46, 0xE2, \ - 0x08, 0xE2, 0x4F, 0xA0, 0x74, 0xE5, 0xAB, 0x31, \ - 0x43, 0xDB, 0x5B, 0xFC, 0xE0, 0xFD, 0x10, 0x8E, \ - 0x4B, 0x82, 0xD1, 0x20, 0xA9, 0x21, 0x08, 0x01, \ - 0x1A, 0x72, 0x3C, 0x12, 0xA7, 0x87, 0xE6, 0xD7, \ - 0x88, 0x71, 0x9A, 0x10, 0xBD, 0xBA, 0x5B, 0x26, \ - 0x99, 0xC3, 0x27, 0x18, 0x6A, 0xF4, 0xE2, 0x3C, \ - 0x1A, 0x94, 0x68, 0x34, 0xB6, 0x15, 0x0B, 0xDA, \ - 0x25, 0x83, 0xE9, 0xCA, 0x2A, 0xD4, 0x4C, 0xE8, \ - 0xDB, 0xBB, 0xC2, 0xDB, 0x04, 0xDE, 0x8E, 0xF9, \ - 0x2E, 0x8E, 0xFC, 0x14, 0x1F, 0xBE, 0xCA, 0xA6, \ - 0x28, 0x7C, 0x59, 0x47, 0x4E, 0x6B, 0xC0, 0x5D, \ - 0x99, 0xB2, 0x96, 0x4F, 0xA0, 0x90, 0xC3, 0xA2, \ - 0x23, 0x3B, 0xA1, 0x86, 0x51, 0x5B, 0xE7, 0xED, \ - 0x1F, 0x61, 0x29, 0x70, 0xCE, 0xE2, 0xD7, 0xAF, \ - 0xB8, 0x1B, 0xDD, 0x76, 0x21, 0x70, 0x48, 0x1C, \ - 0xD0, 0x06, 0x91, 0x27, 0xD5, 0xB0, 0x5A, 0xA9, \ - 0x93, 0xB4, 0xEA, 0x98, 0x8D, 0x8F, 0xDD, 0xC1, \ - 0x86, 0xFF, 0xB7, 0xDC, 0x90, 0xA6, 0xC0, 0x8F, \ - 0x4D, 0xF4, 0x35, 0xC9, 0x34, 0x06, 0x31, 0x99, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC3526_MODP_4096_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC7919_FFDHE2048_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xAD, 0xF8, 0x54, 0x58, 0xA2, 0xBB, 0x4A, 0x9A, \ - 0xAF, 0xDC, 0x56, 0x20, 0x27, 0x3D, 0x3C, 0xF1, \ - 0xD8, 0xB9, 0xC5, 0x83, 0xCE, 0x2D, 0x36, 0x95, \ - 0xA9, 0xE1, 0x36, 0x41, 0x14, 0x64, 0x33, 0xFB, \ - 0xCC, 0x93, 0x9D, 0xCE, 0x24, 0x9B, 0x3E, 0xF9, \ - 0x7D, 0x2F, 0xE3, 0x63, 0x63, 0x0C, 0x75, 0xD8, \ - 0xF6, 0x81, 0xB2, 0x02, 0xAE, 0xC4, 0x61, 0x7A, \ - 0xD3, 0xDF, 0x1E, 0xD5, 0xD5, 0xFD, 0x65, 0x61, \ - 0x24, 0x33, 0xF5, 0x1F, 0x5F, 0x06, 0x6E, 0xD0, \ - 0x85, 0x63, 0x65, 0x55, 0x3D, 0xED, 0x1A, 0xF3, \ - 0xB5, 0x57, 0x13, 0x5E, 0x7F, 0x57, 0xC9, 0x35, \ - 0x98, 0x4F, 0x0C, 0x70, 0xE0, 0xE6, 0x8B, 0x77, \ - 0xE2, 0xA6, 0x89, 0xDA, 0xF3, 0xEF, 0xE8, 0x72, \ - 0x1D, 0xF1, 0x58, 0xA1, 0x36, 0xAD, 0xE7, 0x35, \ - 0x30, 0xAC, 0xCA, 0x4F, 0x48, 0x3A, 0x79, 0x7A, \ - 0xBC, 0x0A, 0xB1, 0x82, 0xB3, 0x24, 0xFB, 0x61, \ - 0xD1, 0x08, 0xA9, 0x4B, 0xB2, 0xC8, 0xE3, 0xFB, \ - 0xB9, 0x6A, 0xDA, 0xB7, 0x60, 0xD7, 0xF4, 0x68, \ - 0x1D, 0x4F, 0x42, 0xA3, 0xDE, 0x39, 0x4D, 0xF4, \ - 0xAE, 0x56, 0xED, 0xE7, 0x63, 0x72, 0xBB, 0x19, \ - 0x0B, 0x07, 0xA7, 0xC8, 0xEE, 0x0A, 0x6D, 0x70, \ - 0x9E, 0x02, 0xFC, 0xE1, 0xCD, 0xF7, 0xE2, 0xEC, \ - 0xC0, 0x34, 0x04, 0xCD, 0x28, 0x34, 0x2F, 0x61, \ - 0x91, 0x72, 0xFE, 0x9C, 0xE9, 0x85, 0x83, 0xFF, \ - 0x8E, 0x4F, 0x12, 0x32, 0xEE, 0xF2, 0x81, 0x83, \ - 0xC3, 0xFE, 0x3B, 0x1B, 0x4C, 0x6F, 0xAD, 0x73, \ - 0x3B, 0xB5, 0xFC, 0xBC, 0x2E, 0xC2, 0x20, 0x05, \ - 0xC5, 0x8E, 0xF1, 0x83, 0x7D, 0x16, 0x83, 0xB2, \ - 0xC6, 0xF3, 0x4A, 0x26, 0xC1, 0xB2, 0xEF, 0xFA, \ - 0x88, 0x6B, 0x42, 0x38, 0x61, 0x28, 0x5C, 0x97, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, } - -#define MBEDTLS_DHM_RFC7919_FFDHE2048_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC7919_FFDHE3072_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xAD, 0xF8, 0x54, 0x58, 0xA2, 0xBB, 0x4A, 0x9A, \ - 0xAF, 0xDC, 0x56, 0x20, 0x27, 0x3D, 0x3C, 0xF1, \ - 0xD8, 0xB9, 0xC5, 0x83, 0xCE, 0x2D, 0x36, 0x95, \ - 0xA9, 0xE1, 0x36, 0x41, 0x14, 0x64, 0x33, 0xFB, \ - 0xCC, 0x93, 0x9D, 0xCE, 0x24, 0x9B, 0x3E, 0xF9, \ - 0x7D, 0x2F, 0xE3, 0x63, 0x63, 0x0C, 0x75, 0xD8, \ - 0xF6, 0x81, 0xB2, 0x02, 0xAE, 0xC4, 0x61, 0x7A, \ - 0xD3, 0xDF, 0x1E, 0xD5, 0xD5, 0xFD, 0x65, 0x61, \ - 0x24, 0x33, 0xF5, 0x1F, 0x5F, 0x06, 0x6E, 0xD0, \ - 0x85, 0x63, 0x65, 0x55, 0x3D, 0xED, 0x1A, 0xF3, \ - 0xB5, 0x57, 0x13, 0x5E, 0x7F, 0x57, 0xC9, 0x35, \ - 0x98, 0x4F, 0x0C, 0x70, 0xE0, 0xE6, 0x8B, 0x77, \ - 0xE2, 0xA6, 0x89, 0xDA, 0xF3, 0xEF, 0xE8, 0x72, \ - 0x1D, 0xF1, 0x58, 0xA1, 0x36, 0xAD, 0xE7, 0x35, \ - 0x30, 0xAC, 0xCA, 0x4F, 0x48, 0x3A, 0x79, 0x7A, \ - 0xBC, 0x0A, 0xB1, 0x82, 0xB3, 0x24, 0xFB, 0x61, \ - 0xD1, 0x08, 0xA9, 0x4B, 0xB2, 0xC8, 0xE3, 0xFB, \ - 0xB9, 0x6A, 0xDA, 0xB7, 0x60, 0xD7, 0xF4, 0x68, \ - 0x1D, 0x4F, 0x42, 0xA3, 0xDE, 0x39, 0x4D, 0xF4, \ - 0xAE, 0x56, 0xED, 0xE7, 0x63, 0x72, 0xBB, 0x19, \ - 0x0B, 0x07, 0xA7, 0xC8, 0xEE, 0x0A, 0x6D, 0x70, \ - 0x9E, 0x02, 0xFC, 0xE1, 0xCD, 0xF7, 0xE2, 0xEC, \ - 0xC0, 0x34, 0x04, 0xCD, 0x28, 0x34, 0x2F, 0x61, \ - 0x91, 0x72, 0xFE, 0x9C, 0xE9, 0x85, 0x83, 0xFF, \ - 0x8E, 0x4F, 0x12, 0x32, 0xEE, 0xF2, 0x81, 0x83, \ - 0xC3, 0xFE, 0x3B, 0x1B, 0x4C, 0x6F, 0xAD, 0x73, \ - 0x3B, 0xB5, 0xFC, 0xBC, 0x2E, 0xC2, 0x20, 0x05, \ - 0xC5, 0x8E, 0xF1, 0x83, 0x7D, 0x16, 0x83, 0xB2, \ - 0xC6, 0xF3, 0x4A, 0x26, 0xC1, 0xB2, 0xEF, 0xFA, \ - 0x88, 0x6B, 0x42, 0x38, 0x61, 0x1F, 0xCF, 0xDC, \ - 0xDE, 0x35, 0x5B, 0x3B, 0x65, 0x19, 0x03, 0x5B, \ - 0xBC, 0x34, 0xF4, 0xDE, 0xF9, 0x9C, 0x02, 0x38, \ - 0x61, 0xB4, 0x6F, 0xC9, 0xD6, 0xE6, 0xC9, 0x07, \ - 0x7A, 0xD9, 0x1D, 0x26, 0x91, 0xF7, 0xF7, 0xEE, \ - 0x59, 0x8C, 0xB0, 0xFA, 0xC1, 0x86, 0xD9, 0x1C, \ - 0xAE, 0xFE, 0x13, 0x09, 0x85, 0x13, 0x92, 0x70, \ - 0xB4, 0x13, 0x0C, 0x93, 0xBC, 0x43, 0x79, 0x44, \ - 0xF4, 0xFD, 0x44, 0x52, 0xE2, 0xD7, 0x4D, 0xD3, \ - 0x64, 0xF2, 0xE2, 0x1E, 0x71, 0xF5, 0x4B, 0xFF, \ - 0x5C, 0xAE, 0x82, 0xAB, 0x9C, 0x9D, 0xF6, 0x9E, \ - 0xE8, 0x6D, 0x2B, 0xC5, 0x22, 0x36, 0x3A, 0x0D, \ - 0xAB, 0xC5, 0x21, 0x97, 0x9B, 0x0D, 0xEA, 0xDA, \ - 0x1D, 0xBF, 0x9A, 0x42, 0xD5, 0xC4, 0x48, 0x4E, \ - 0x0A, 0xBC, 0xD0, 0x6B, 0xFA, 0x53, 0xDD, 0xEF, \ - 0x3C, 0x1B, 0x20, 0xEE, 0x3F, 0xD5, 0x9D, 0x7C, \ - 0x25, 0xE4, 0x1D, 0x2B, 0x66, 0xC6, 0x2E, 0x37, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC7919_FFDHE3072_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC7919_FFDHE4096_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xAD, 0xF8, 0x54, 0x58, 0xA2, 0xBB, 0x4A, 0x9A, \ - 0xAF, 0xDC, 0x56, 0x20, 0x27, 0x3D, 0x3C, 0xF1, \ - 0xD8, 0xB9, 0xC5, 0x83, 0xCE, 0x2D, 0x36, 0x95, \ - 0xA9, 0xE1, 0x36, 0x41, 0x14, 0x64, 0x33, 0xFB, \ - 0xCC, 0x93, 0x9D, 0xCE, 0x24, 0x9B, 0x3E, 0xF9, \ - 0x7D, 0x2F, 0xE3, 0x63, 0x63, 0x0C, 0x75, 0xD8, \ - 0xF6, 0x81, 0xB2, 0x02, 0xAE, 0xC4, 0x61, 0x7A, \ - 0xD3, 0xDF, 0x1E, 0xD5, 0xD5, 0xFD, 0x65, 0x61, \ - 0x24, 0x33, 0xF5, 0x1F, 0x5F, 0x06, 0x6E, 0xD0, \ - 0x85, 0x63, 0x65, 0x55, 0x3D, 0xED, 0x1A, 0xF3, \ - 0xB5, 0x57, 0x13, 0x5E, 0x7F, 0x57, 0xC9, 0x35, \ - 0x98, 0x4F, 0x0C, 0x70, 0xE0, 0xE6, 0x8B, 0x77, \ - 0xE2, 0xA6, 0x89, 0xDA, 0xF3, 0xEF, 0xE8, 0x72, \ - 0x1D, 0xF1, 0x58, 0xA1, 0x36, 0xAD, 0xE7, 0x35, \ - 0x30, 0xAC, 0xCA, 0x4F, 0x48, 0x3A, 0x79, 0x7A, \ - 0xBC, 0x0A, 0xB1, 0x82, 0xB3, 0x24, 0xFB, 0x61, \ - 0xD1, 0x08, 0xA9, 0x4B, 0xB2, 0xC8, 0xE3, 0xFB, \ - 0xB9, 0x6A, 0xDA, 0xB7, 0x60, 0xD7, 0xF4, 0x68, \ - 0x1D, 0x4F, 0x42, 0xA3, 0xDE, 0x39, 0x4D, 0xF4, \ - 0xAE, 0x56, 0xED, 0xE7, 0x63, 0x72, 0xBB, 0x19, \ - 0x0B, 0x07, 0xA7, 0xC8, 0xEE, 0x0A, 0x6D, 0x70, \ - 0x9E, 0x02, 0xFC, 0xE1, 0xCD, 0xF7, 0xE2, 0xEC, \ - 0xC0, 0x34, 0x04, 0xCD, 0x28, 0x34, 0x2F, 0x61, \ - 0x91, 0x72, 0xFE, 0x9C, 0xE9, 0x85, 0x83, 0xFF, \ - 0x8E, 0x4F, 0x12, 0x32, 0xEE, 0xF2, 0x81, 0x83, \ - 0xC3, 0xFE, 0x3B, 0x1B, 0x4C, 0x6F, 0xAD, 0x73, \ - 0x3B, 0xB5, 0xFC, 0xBC, 0x2E, 0xC2, 0x20, 0x05, \ - 0xC5, 0x8E, 0xF1, 0x83, 0x7D, 0x16, 0x83, 0xB2, \ - 0xC6, 0xF3, 0x4A, 0x26, 0xC1, 0xB2, 0xEF, 0xFA, \ - 0x88, 0x6B, 0x42, 0x38, 0x61, 0x1F, 0xCF, 0xDC, \ - 0xDE, 0x35, 0x5B, 0x3B, 0x65, 0x19, 0x03, 0x5B, \ - 0xBC, 0x34, 0xF4, 0xDE, 0xF9, 0x9C, 0x02, 0x38, \ - 0x61, 0xB4, 0x6F, 0xC9, 0xD6, 0xE6, 0xC9, 0x07, \ - 0x7A, 0xD9, 0x1D, 0x26, 0x91, 0xF7, 0xF7, 0xEE, \ - 0x59, 0x8C, 0xB0, 0xFA, 0xC1, 0x86, 0xD9, 0x1C, \ - 0xAE, 0xFE, 0x13, 0x09, 0x85, 0x13, 0x92, 0x70, \ - 0xB4, 0x13, 0x0C, 0x93, 0xBC, 0x43, 0x79, 0x44, \ - 0xF4, 0xFD, 0x44, 0x52, 0xE2, 0xD7, 0x4D, 0xD3, \ - 0x64, 0xF2, 0xE2, 0x1E, 0x71, 0xF5, 0x4B, 0xFF, \ - 0x5C, 0xAE, 0x82, 0xAB, 0x9C, 0x9D, 0xF6, 0x9E, \ - 0xE8, 0x6D, 0x2B, 0xC5, 0x22, 0x36, 0x3A, 0x0D, \ - 0xAB, 0xC5, 0x21, 0x97, 0x9B, 0x0D, 0xEA, 0xDA, \ - 0x1D, 0xBF, 0x9A, 0x42, 0xD5, 0xC4, 0x48, 0x4E, \ - 0x0A, 0xBC, 0xD0, 0x6B, 0xFA, 0x53, 0xDD, 0xEF, \ - 0x3C, 0x1B, 0x20, 0xEE, 0x3F, 0xD5, 0x9D, 0x7C, \ - 0x25, 0xE4, 0x1D, 0x2B, 0x66, 0x9E, 0x1E, 0xF1, \ - 0x6E, 0x6F, 0x52, 0xC3, 0x16, 0x4D, 0xF4, 0xFB, \ - 0x79, 0x30, 0xE9, 0xE4, 0xE5, 0x88, 0x57, 0xB6, \ - 0xAC, 0x7D, 0x5F, 0x42, 0xD6, 0x9F, 0x6D, 0x18, \ - 0x77, 0x63, 0xCF, 0x1D, 0x55, 0x03, 0x40, 0x04, \ - 0x87, 0xF5, 0x5B, 0xA5, 0x7E, 0x31, 0xCC, 0x7A, \ - 0x71, 0x35, 0xC8, 0x86, 0xEF, 0xB4, 0x31, 0x8A, \ - 0xED, 0x6A, 0x1E, 0x01, 0x2D, 0x9E, 0x68, 0x32, \ - 0xA9, 0x07, 0x60, 0x0A, 0x91, 0x81, 0x30, 0xC4, \ - 0x6D, 0xC7, 0x78, 0xF9, 0x71, 0xAD, 0x00, 0x38, \ - 0x09, 0x29, 0x99, 0xA3, 0x33, 0xCB, 0x8B, 0x7A, \ - 0x1A, 0x1D, 0xB9, 0x3D, 0x71, 0x40, 0x00, 0x3C, \ - 0x2A, 0x4E, 0xCE, 0xA9, 0xF9, 0x8D, 0x0A, 0xCC, \ - 0x0A, 0x82, 0x91, 0xCD, 0xCE, 0xC9, 0x7D, 0xCF, \ - 0x8E, 0xC9, 0xB5, 0x5A, 0x7F, 0x88, 0xA4, 0x6B, \ - 0x4D, 0xB5, 0xA8, 0x51, 0xF4, 0x41, 0x82, 0xE1, \ - 0xC6, 0x8A, 0x00, 0x7E, 0x5E, 0x65, 0x5F, 0x6A, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC7919_FFDHE4096_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC7919_FFDHE6144_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xAD, 0xF8, 0x54, 0x58, 0xA2, 0xBB, 0x4A, 0x9A, \ - 0xAF, 0xDC, 0x56, 0x20, 0x27, 0x3D, 0x3C, 0xF1, \ - 0xD8, 0xB9, 0xC5, 0x83, 0xCE, 0x2D, 0x36, 0x95, \ - 0xA9, 0xE1, 0x36, 0x41, 0x14, 0x64, 0x33, 0xFB, \ - 0xCC, 0x93, 0x9D, 0xCE, 0x24, 0x9B, 0x3E, 0xF9, \ - 0x7D, 0x2F, 0xE3, 0x63, 0x63, 0x0C, 0x75, 0xD8, \ - 0xF6, 0x81, 0xB2, 0x02, 0xAE, 0xC4, 0x61, 0x7A, \ - 0xD3, 0xDF, 0x1E, 0xD5, 0xD5, 0xFD, 0x65, 0x61, \ - 0x24, 0x33, 0xF5, 0x1F, 0x5F, 0x06, 0x6E, 0xD0, \ - 0x85, 0x63, 0x65, 0x55, 0x3D, 0xED, 0x1A, 0xF3, \ - 0xB5, 0x57, 0x13, 0x5E, 0x7F, 0x57, 0xC9, 0x35, \ - 0x98, 0x4F, 0x0C, 0x70, 0xE0, 0xE6, 0x8B, 0x77, \ - 0xE2, 0xA6, 0x89, 0xDA, 0xF3, 0xEF, 0xE8, 0x72, \ - 0x1D, 0xF1, 0x58, 0xA1, 0x36, 0xAD, 0xE7, 0x35, \ - 0x30, 0xAC, 0xCA, 0x4F, 0x48, 0x3A, 0x79, 0x7A, \ - 0xBC, 0x0A, 0xB1, 0x82, 0xB3, 0x24, 0xFB, 0x61, \ - 0xD1, 0x08, 0xA9, 0x4B, 0xB2, 0xC8, 0xE3, 0xFB, \ - 0xB9, 0x6A, 0xDA, 0xB7, 0x60, 0xD7, 0xF4, 0x68, \ - 0x1D, 0x4F, 0x42, 0xA3, 0xDE, 0x39, 0x4D, 0xF4, \ - 0xAE, 0x56, 0xED, 0xE7, 0x63, 0x72, 0xBB, 0x19, \ - 0x0B, 0x07, 0xA7, 0xC8, 0xEE, 0x0A, 0x6D, 0x70, \ - 0x9E, 0x02, 0xFC, 0xE1, 0xCD, 0xF7, 0xE2, 0xEC, \ - 0xC0, 0x34, 0x04, 0xCD, 0x28, 0x34, 0x2F, 0x61, \ - 0x91, 0x72, 0xFE, 0x9C, 0xE9, 0x85, 0x83, 0xFF, \ - 0x8E, 0x4F, 0x12, 0x32, 0xEE, 0xF2, 0x81, 0x83, \ - 0xC3, 0xFE, 0x3B, 0x1B, 0x4C, 0x6F, 0xAD, 0x73, \ - 0x3B, 0xB5, 0xFC, 0xBC, 0x2E, 0xC2, 0x20, 0x05, \ - 0xC5, 0x8E, 0xF1, 0x83, 0x7D, 0x16, 0x83, 0xB2, \ - 0xC6, 0xF3, 0x4A, 0x26, 0xC1, 0xB2, 0xEF, 0xFA, \ - 0x88, 0x6B, 0x42, 0x38, 0x61, 0x1F, 0xCF, 0xDC, \ - 0xDE, 0x35, 0x5B, 0x3B, 0x65, 0x19, 0x03, 0x5B, \ - 0xBC, 0x34, 0xF4, 0xDE, 0xF9, 0x9C, 0x02, 0x38, \ - 0x61, 0xB4, 0x6F, 0xC9, 0xD6, 0xE6, 0xC9, 0x07, \ - 0x7A, 0xD9, 0x1D, 0x26, 0x91, 0xF7, 0xF7, 0xEE, \ - 0x59, 0x8C, 0xB0, 0xFA, 0xC1, 0x86, 0xD9, 0x1C, \ - 0xAE, 0xFE, 0x13, 0x09, 0x85, 0x13, 0x92, 0x70, \ - 0xB4, 0x13, 0x0C, 0x93, 0xBC, 0x43, 0x79, 0x44, \ - 0xF4, 0xFD, 0x44, 0x52, 0xE2, 0xD7, 0x4D, 0xD3, \ - 0x64, 0xF2, 0xE2, 0x1E, 0x71, 0xF5, 0x4B, 0xFF, \ - 0x5C, 0xAE, 0x82, 0xAB, 0x9C, 0x9D, 0xF6, 0x9E, \ - 0xE8, 0x6D, 0x2B, 0xC5, 0x22, 0x36, 0x3A, 0x0D, \ - 0xAB, 0xC5, 0x21, 0x97, 0x9B, 0x0D, 0xEA, 0xDA, \ - 0x1D, 0xBF, 0x9A, 0x42, 0xD5, 0xC4, 0x48, 0x4E, \ - 0x0A, 0xBC, 0xD0, 0x6B, 0xFA, 0x53, 0xDD, 0xEF, \ - 0x3C, 0x1B, 0x20, 0xEE, 0x3F, 0xD5, 0x9D, 0x7C, \ - 0x25, 0xE4, 0x1D, 0x2B, 0x66, 0x9E, 0x1E, 0xF1, \ - 0x6E, 0x6F, 0x52, 0xC3, 0x16, 0x4D, 0xF4, 0xFB, \ - 0x79, 0x30, 0xE9, 0xE4, 0xE5, 0x88, 0x57, 0xB6, \ - 0xAC, 0x7D, 0x5F, 0x42, 0xD6, 0x9F, 0x6D, 0x18, \ - 0x77, 0x63, 0xCF, 0x1D, 0x55, 0x03, 0x40, 0x04, \ - 0x87, 0xF5, 0x5B, 0xA5, 0x7E, 0x31, 0xCC, 0x7A, \ - 0x71, 0x35, 0xC8, 0x86, 0xEF, 0xB4, 0x31, 0x8A, \ - 0xED, 0x6A, 0x1E, 0x01, 0x2D, 0x9E, 0x68, 0x32, \ - 0xA9, 0x07, 0x60, 0x0A, 0x91, 0x81, 0x30, 0xC4, \ - 0x6D, 0xC7, 0x78, 0xF9, 0x71, 0xAD, 0x00, 0x38, \ - 0x09, 0x29, 0x99, 0xA3, 0x33, 0xCB, 0x8B, 0x7A, \ - 0x1A, 0x1D, 0xB9, 0x3D, 0x71, 0x40, 0x00, 0x3C, \ - 0x2A, 0x4E, 0xCE, 0xA9, 0xF9, 0x8D, 0x0A, 0xCC, \ - 0x0A, 0x82, 0x91, 0xCD, 0xCE, 0xC9, 0x7D, 0xCF, \ - 0x8E, 0xC9, 0xB5, 0x5A, 0x7F, 0x88, 0xA4, 0x6B, \ - 0x4D, 0xB5, 0xA8, 0x51, 0xF4, 0x41, 0x82, 0xE1, \ - 0xC6, 0x8A, 0x00, 0x7E, 0x5E, 0x0D, 0xD9, 0x02, \ - 0x0B, 0xFD, 0x64, 0xB6, 0x45, 0x03, 0x6C, 0x7A, \ - 0x4E, 0x67, 0x7D, 0x2C, 0x38, 0x53, 0x2A, 0x3A, \ - 0x23, 0xBA, 0x44, 0x42, 0xCA, 0xF5, 0x3E, 0xA6, \ - 0x3B, 0xB4, 0x54, 0x32, 0x9B, 0x76, 0x24, 0xC8, \ - 0x91, 0x7B, 0xDD, 0x64, 0xB1, 0xC0, 0xFD, 0x4C, \ - 0xB3, 0x8E, 0x8C, 0x33, 0x4C, 0x70, 0x1C, 0x3A, \ - 0xCD, 0xAD, 0x06, 0x57, 0xFC, 0xCF, 0xEC, 0x71, \ - 0x9B, 0x1F, 0x5C, 0x3E, 0x4E, 0x46, 0x04, 0x1F, \ - 0x38, 0x81, 0x47, 0xFB, 0x4C, 0xFD, 0xB4, 0x77, \ - 0xA5, 0x24, 0x71, 0xF7, 0xA9, 0xA9, 0x69, 0x10, \ - 0xB8, 0x55, 0x32, 0x2E, 0xDB, 0x63, 0x40, 0xD8, \ - 0xA0, 0x0E, 0xF0, 0x92, 0x35, 0x05, 0x11, 0xE3, \ - 0x0A, 0xBE, 0xC1, 0xFF, 0xF9, 0xE3, 0xA2, 0x6E, \ - 0x7F, 0xB2, 0x9F, 0x8C, 0x18, 0x30, 0x23, 0xC3, \ - 0x58, 0x7E, 0x38, 0xDA, 0x00, 0x77, 0xD9, 0xB4, \ - 0x76, 0x3E, 0x4E, 0x4B, 0x94, 0xB2, 0xBB, 0xC1, \ - 0x94, 0xC6, 0x65, 0x1E, 0x77, 0xCA, 0xF9, 0x92, \ - 0xEE, 0xAA, 0xC0, 0x23, 0x2A, 0x28, 0x1B, 0xF6, \ - 0xB3, 0xA7, 0x39, 0xC1, 0x22, 0x61, 0x16, 0x82, \ - 0x0A, 0xE8, 0xDB, 0x58, 0x47, 0xA6, 0x7C, 0xBE, \ - 0xF9, 0xC9, 0x09, 0x1B, 0x46, 0x2D, 0x53, 0x8C, \ - 0xD7, 0x2B, 0x03, 0x74, 0x6A, 0xE7, 0x7F, 0x5E, \ - 0x62, 0x29, 0x2C, 0x31, 0x15, 0x62, 0xA8, 0x46, \ - 0x50, 0x5D, 0xC8, 0x2D, 0xB8, 0x54, 0x33, 0x8A, \ - 0xE4, 0x9F, 0x52, 0x35, 0xC9, 0x5B, 0x91, 0x17, \ - 0x8C, 0xCF, 0x2D, 0xD5, 0xCA, 0xCE, 0xF4, 0x03, \ - 0xEC, 0x9D, 0x18, 0x10, 0xC6, 0x27, 0x2B, 0x04, \ - 0x5B, 0x3B, 0x71, 0xF9, 0xDC, 0x6B, 0x80, 0xD6, \ - 0x3F, 0xDD, 0x4A, 0x8E, 0x9A, 0xDB, 0x1E, 0x69, \ - 0x62, 0xA6, 0x95, 0x26, 0xD4, 0x31, 0x61, 0xC1, \ - 0xA4, 0x1D, 0x57, 0x0D, 0x79, 0x38, 0xDA, 0xD4, \ - 0xA4, 0x0E, 0x32, 0x9C, 0xD0, 0xE4, 0x0E, 0x65, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC7919_FFDHE6144_G_BIN { 0x02 } - -#define MBEDTLS_DHM_RFC7919_FFDHE8192_P_BIN { \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, \ - 0xAD, 0xF8, 0x54, 0x58, 0xA2, 0xBB, 0x4A, 0x9A, \ - 0xAF, 0xDC, 0x56, 0x20, 0x27, 0x3D, 0x3C, 0xF1, \ - 0xD8, 0xB9, 0xC5, 0x83, 0xCE, 0x2D, 0x36, 0x95, \ - 0xA9, 0xE1, 0x36, 0x41, 0x14, 0x64, 0x33, 0xFB, \ - 0xCC, 0x93, 0x9D, 0xCE, 0x24, 0x9B, 0x3E, 0xF9, \ - 0x7D, 0x2F, 0xE3, 0x63, 0x63, 0x0C, 0x75, 0xD8, \ - 0xF6, 0x81, 0xB2, 0x02, 0xAE, 0xC4, 0x61, 0x7A, \ - 0xD3, 0xDF, 0x1E, 0xD5, 0xD5, 0xFD, 0x65, 0x61, \ - 0x24, 0x33, 0xF5, 0x1F, 0x5F, 0x06, 0x6E, 0xD0, \ - 0x85, 0x63, 0x65, 0x55, 0x3D, 0xED, 0x1A, 0xF3, \ - 0xB5, 0x57, 0x13, 0x5E, 0x7F, 0x57, 0xC9, 0x35, \ - 0x98, 0x4F, 0x0C, 0x70, 0xE0, 0xE6, 0x8B, 0x77, \ - 0xE2, 0xA6, 0x89, 0xDA, 0xF3, 0xEF, 0xE8, 0x72, \ - 0x1D, 0xF1, 0x58, 0xA1, 0x36, 0xAD, 0xE7, 0x35, \ - 0x30, 0xAC, 0xCA, 0x4F, 0x48, 0x3A, 0x79, 0x7A, \ - 0xBC, 0x0A, 0xB1, 0x82, 0xB3, 0x24, 0xFB, 0x61, \ - 0xD1, 0x08, 0xA9, 0x4B, 0xB2, 0xC8, 0xE3, 0xFB, \ - 0xB9, 0x6A, 0xDA, 0xB7, 0x60, 0xD7, 0xF4, 0x68, \ - 0x1D, 0x4F, 0x42, 0xA3, 0xDE, 0x39, 0x4D, 0xF4, \ - 0xAE, 0x56, 0xED, 0xE7, 0x63, 0x72, 0xBB, 0x19, \ - 0x0B, 0x07, 0xA7, 0xC8, 0xEE, 0x0A, 0x6D, 0x70, \ - 0x9E, 0x02, 0xFC, 0xE1, 0xCD, 0xF7, 0xE2, 0xEC, \ - 0xC0, 0x34, 0x04, 0xCD, 0x28, 0x34, 0x2F, 0x61, \ - 0x91, 0x72, 0xFE, 0x9C, 0xE9, 0x85, 0x83, 0xFF, \ - 0x8E, 0x4F, 0x12, 0x32, 0xEE, 0xF2, 0x81, 0x83, \ - 0xC3, 0xFE, 0x3B, 0x1B, 0x4C, 0x6F, 0xAD, 0x73, \ - 0x3B, 0xB5, 0xFC, 0xBC, 0x2E, 0xC2, 0x20, 0x05, \ - 0xC5, 0x8E, 0xF1, 0x83, 0x7D, 0x16, 0x83, 0xB2, \ - 0xC6, 0xF3, 0x4A, 0x26, 0xC1, 0xB2, 0xEF, 0xFA, \ - 0x88, 0x6B, 0x42, 0x38, 0x61, 0x1F, 0xCF, 0xDC, \ - 0xDE, 0x35, 0x5B, 0x3B, 0x65, 0x19, 0x03, 0x5B, \ - 0xBC, 0x34, 0xF4, 0xDE, 0xF9, 0x9C, 0x02, 0x38, \ - 0x61, 0xB4, 0x6F, 0xC9, 0xD6, 0xE6, 0xC9, 0x07, \ - 0x7A, 0xD9, 0x1D, 0x26, 0x91, 0xF7, 0xF7, 0xEE, \ - 0x59, 0x8C, 0xB0, 0xFA, 0xC1, 0x86, 0xD9, 0x1C, \ - 0xAE, 0xFE, 0x13, 0x09, 0x85, 0x13, 0x92, 0x70, \ - 0xB4, 0x13, 0x0C, 0x93, 0xBC, 0x43, 0x79, 0x44, \ - 0xF4, 0xFD, 0x44, 0x52, 0xE2, 0xD7, 0x4D, 0xD3, \ - 0x64, 0xF2, 0xE2, 0x1E, 0x71, 0xF5, 0x4B, 0xFF, \ - 0x5C, 0xAE, 0x82, 0xAB, 0x9C, 0x9D, 0xF6, 0x9E, \ - 0xE8, 0x6D, 0x2B, 0xC5, 0x22, 0x36, 0x3A, 0x0D, \ - 0xAB, 0xC5, 0x21, 0x97, 0x9B, 0x0D, 0xEA, 0xDA, \ - 0x1D, 0xBF, 0x9A, 0x42, 0xD5, 0xC4, 0x48, 0x4E, \ - 0x0A, 0xBC, 0xD0, 0x6B, 0xFA, 0x53, 0xDD, 0xEF, \ - 0x3C, 0x1B, 0x20, 0xEE, 0x3F, 0xD5, 0x9D, 0x7C, \ - 0x25, 0xE4, 0x1D, 0x2B, 0x66, 0x9E, 0x1E, 0xF1, \ - 0x6E, 0x6F, 0x52, 0xC3, 0x16, 0x4D, 0xF4, 0xFB, \ - 0x79, 0x30, 0xE9, 0xE4, 0xE5, 0x88, 0x57, 0xB6, \ - 0xAC, 0x7D, 0x5F, 0x42, 0xD6, 0x9F, 0x6D, 0x18, \ - 0x77, 0x63, 0xCF, 0x1D, 0x55, 0x03, 0x40, 0x04, \ - 0x87, 0xF5, 0x5B, 0xA5, 0x7E, 0x31, 0xCC, 0x7A, \ - 0x71, 0x35, 0xC8, 0x86, 0xEF, 0xB4, 0x31, 0x8A, \ - 0xED, 0x6A, 0x1E, 0x01, 0x2D, 0x9E, 0x68, 0x32, \ - 0xA9, 0x07, 0x60, 0x0A, 0x91, 0x81, 0x30, 0xC4, \ - 0x6D, 0xC7, 0x78, 0xF9, 0x71, 0xAD, 0x00, 0x38, \ - 0x09, 0x29, 0x99, 0xA3, 0x33, 0xCB, 0x8B, 0x7A, \ - 0x1A, 0x1D, 0xB9, 0x3D, 0x71, 0x40, 0x00, 0x3C, \ - 0x2A, 0x4E, 0xCE, 0xA9, 0xF9, 0x8D, 0x0A, 0xCC, \ - 0x0A, 0x82, 0x91, 0xCD, 0xCE, 0xC9, 0x7D, 0xCF, \ - 0x8E, 0xC9, 0xB5, 0x5A, 0x7F, 0x88, 0xA4, 0x6B, \ - 0x4D, 0xB5, 0xA8, 0x51, 0xF4, 0x41, 0x82, 0xE1, \ - 0xC6, 0x8A, 0x00, 0x7E, 0x5E, 0x0D, 0xD9, 0x02, \ - 0x0B, 0xFD, 0x64, 0xB6, 0x45, 0x03, 0x6C, 0x7A, \ - 0x4E, 0x67, 0x7D, 0x2C, 0x38, 0x53, 0x2A, 0x3A, \ - 0x23, 0xBA, 0x44, 0x42, 0xCA, 0xF5, 0x3E, 0xA6, \ - 0x3B, 0xB4, 0x54, 0x32, 0x9B, 0x76, 0x24, 0xC8, \ - 0x91, 0x7B, 0xDD, 0x64, 0xB1, 0xC0, 0xFD, 0x4C, \ - 0xB3, 0x8E, 0x8C, 0x33, 0x4C, 0x70, 0x1C, 0x3A, \ - 0xCD, 0xAD, 0x06, 0x57, 0xFC, 0xCF, 0xEC, 0x71, \ - 0x9B, 0x1F, 0x5C, 0x3E, 0x4E, 0x46, 0x04, 0x1F, \ - 0x38, 0x81, 0x47, 0xFB, 0x4C, 0xFD, 0xB4, 0x77, \ - 0xA5, 0x24, 0x71, 0xF7, 0xA9, 0xA9, 0x69, 0x10, \ - 0xB8, 0x55, 0x32, 0x2E, 0xDB, 0x63, 0x40, 0xD8, \ - 0xA0, 0x0E, 0xF0, 0x92, 0x35, 0x05, 0x11, 0xE3, \ - 0x0A, 0xBE, 0xC1, 0xFF, 0xF9, 0xE3, 0xA2, 0x6E, \ - 0x7F, 0xB2, 0x9F, 0x8C, 0x18, 0x30, 0x23, 0xC3, \ - 0x58, 0x7E, 0x38, 0xDA, 0x00, 0x77, 0xD9, 0xB4, \ - 0x76, 0x3E, 0x4E, 0x4B, 0x94, 0xB2, 0xBB, 0xC1, \ - 0x94, 0xC6, 0x65, 0x1E, 0x77, 0xCA, 0xF9, 0x92, \ - 0xEE, 0xAA, 0xC0, 0x23, 0x2A, 0x28, 0x1B, 0xF6, \ - 0xB3, 0xA7, 0x39, 0xC1, 0x22, 0x61, 0x16, 0x82, \ - 0x0A, 0xE8, 0xDB, 0x58, 0x47, 0xA6, 0x7C, 0xBE, \ - 0xF9, 0xC9, 0x09, 0x1B, 0x46, 0x2D, 0x53, 0x8C, \ - 0xD7, 0x2B, 0x03, 0x74, 0x6A, 0xE7, 0x7F, 0x5E, \ - 0x62, 0x29, 0x2C, 0x31, 0x15, 0x62, 0xA8, 0x46, \ - 0x50, 0x5D, 0xC8, 0x2D, 0xB8, 0x54, 0x33, 0x8A, \ - 0xE4, 0x9F, 0x52, 0x35, 0xC9, 0x5B, 0x91, 0x17, \ - 0x8C, 0xCF, 0x2D, 0xD5, 0xCA, 0xCE, 0xF4, 0x03, \ - 0xEC, 0x9D, 0x18, 0x10, 0xC6, 0x27, 0x2B, 0x04, \ - 0x5B, 0x3B, 0x71, 0xF9, 0xDC, 0x6B, 0x80, 0xD6, \ - 0x3F, 0xDD, 0x4A, 0x8E, 0x9A, 0xDB, 0x1E, 0x69, \ - 0x62, 0xA6, 0x95, 0x26, 0xD4, 0x31, 0x61, 0xC1, \ - 0xA4, 0x1D, 0x57, 0x0D, 0x79, 0x38, 0xDA, 0xD4, \ - 0xA4, 0x0E, 0x32, 0x9C, 0xCF, 0xF4, 0x6A, 0xAA, \ - 0x36, 0xAD, 0x00, 0x4C, 0xF6, 0x00, 0xC8, 0x38, \ - 0x1E, 0x42, 0x5A, 0x31, 0xD9, 0x51, 0xAE, 0x64, \ - 0xFD, 0xB2, 0x3F, 0xCE, 0xC9, 0x50, 0x9D, 0x43, \ - 0x68, 0x7F, 0xEB, 0x69, 0xED, 0xD1, 0xCC, 0x5E, \ - 0x0B, 0x8C, 0xC3, 0xBD, 0xF6, 0x4B, 0x10, 0xEF, \ - 0x86, 0xB6, 0x31, 0x42, 0xA3, 0xAB, 0x88, 0x29, \ - 0x55, 0x5B, 0x2F, 0x74, 0x7C, 0x93, 0x26, 0x65, \ - 0xCB, 0x2C, 0x0F, 0x1C, 0xC0, 0x1B, 0xD7, 0x02, \ - 0x29, 0x38, 0x88, 0x39, 0xD2, 0xAF, 0x05, 0xE4, \ - 0x54, 0x50, 0x4A, 0xC7, 0x8B, 0x75, 0x82, 0x82, \ - 0x28, 0x46, 0xC0, 0xBA, 0x35, 0xC3, 0x5F, 0x5C, \ - 0x59, 0x16, 0x0C, 0xC0, 0x46, 0xFD, 0x82, 0x51, \ - 0x54, 0x1F, 0xC6, 0x8C, 0x9C, 0x86, 0xB0, 0x22, \ - 0xBB, 0x70, 0x99, 0x87, 0x6A, 0x46, 0x0E, 0x74, \ - 0x51, 0xA8, 0xA9, 0x31, 0x09, 0x70, 0x3F, 0xEE, \ - 0x1C, 0x21, 0x7E, 0x6C, 0x38, 0x26, 0xE5, 0x2C, \ - 0x51, 0xAA, 0x69, 0x1E, 0x0E, 0x42, 0x3C, 0xFC, \ - 0x99, 0xE9, 0xE3, 0x16, 0x50, 0xC1, 0x21, 0x7B, \ - 0x62, 0x48, 0x16, 0xCD, 0xAD, 0x9A, 0x95, 0xF9, \ - 0xD5, 0xB8, 0x01, 0x94, 0x88, 0xD9, 0xC0, 0xA0, \ - 0xA1, 0xFE, 0x30, 0x75, 0xA5, 0x77, 0xE2, 0x31, \ - 0x83, 0xF8, 0x1D, 0x4A, 0x3F, 0x2F, 0xA4, 0x57, \ - 0x1E, 0xFC, 0x8C, 0xE0, 0xBA, 0x8A, 0x4F, 0xE8, \ - 0xB6, 0x85, 0x5D, 0xFE, 0x72, 0xB0, 0xA6, 0x6E, \ - 0xDE, 0xD2, 0xFB, 0xAB, 0xFB, 0xE5, 0x8A, 0x30, \ - 0xFA, 0xFA, 0xBE, 0x1C, 0x5D, 0x71, 0xA8, 0x7E, \ - 0x2F, 0x74, 0x1E, 0xF8, 0xC1, 0xFE, 0x86, 0xFE, \ - 0xA6, 0xBB, 0xFD, 0xE5, 0x30, 0x67, 0x7F, 0x0D, \ - 0x97, 0xD1, 0x1D, 0x49, 0xF7, 0xA8, 0x44, 0x3D, \ - 0x08, 0x22, 0xE5, 0x06, 0xA9, 0xF4, 0x61, 0x4E, \ - 0x01, 0x1E, 0x2A, 0x94, 0x83, 0x8F, 0xF8, 0x8C, \ - 0xD6, 0x8C, 0x8B, 0xB7, 0xC5, 0xC6, 0x42, 0x4C, \ - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF } - -#define MBEDTLS_DHM_RFC7919_FFDHE8192_G_BIN { 0x02 } - -#endif /* dhm.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ecdh.h b/android/mbedtls/arm64-v8a/include/mbedtls/ecdh.h deleted file mode 100644 index a919ad27e..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ecdh.h +++ /dev/null @@ -1,454 +0,0 @@ -/** - * \file ecdh.h - * - * \brief This file contains ECDH definitions and functions. - * - * The Elliptic Curve Diffie-Hellman (ECDH) protocol is an anonymous - * key agreement protocol allowing two parties to establish a shared - * secret over an insecure channel. Each party must have an - * elliptic-curve public–private key pair. - * - * For more information, see NIST SP 800-56A Rev. 2: Recommendation for - * Pair-Wise Key Establishment Schemes Using Discrete Logarithm - * Cryptography. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_ECDH_H -#define MBEDTLS_ECDH_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/ecp.h" - -/* - * Mbed TLS supports two formats for ECDH contexts (#mbedtls_ecdh_context - * defined in `ecdh.h`). For most applications, the choice of format makes - * no difference, since all library functions can work with either format, - * except that the new format is incompatible with MBEDTLS_ECP_RESTARTABLE. - - * The new format used when this option is disabled is smaller - * (56 bytes on a 32-bit platform). In future versions of the library, it - * will support alternative implementations of ECDH operations. - * The new format is incompatible with applications that access - * context fields directly and with restartable ECP operations. - */ - -#if defined(MBEDTLS_ECP_RESTARTABLE) -#define MBEDTLS_ECDH_LEGACY_CONTEXT -#else -#undef MBEDTLS_ECDH_LEGACY_CONTEXT -#endif - -#if defined(MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED) -#undef MBEDTLS_ECDH_LEGACY_CONTEXT -#include "everest/everest.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Defines the source of the imported EC key. - */ -typedef enum { - MBEDTLS_ECDH_OURS, /**< Our key. */ - MBEDTLS_ECDH_THEIRS, /**< The key of the peer. */ -} mbedtls_ecdh_side; - -#if !defined(MBEDTLS_ECDH_LEGACY_CONTEXT) -/** - * Defines the ECDH implementation used. - * - * Later versions of the library may add new variants, therefore users should - * not make any assumptions about them. - */ -typedef enum { - MBEDTLS_ECDH_VARIANT_NONE = 0, /*!< Implementation not defined. */ - MBEDTLS_ECDH_VARIANT_MBEDTLS_2_0,/*!< The default Mbed TLS implementation */ -#if defined(MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED) - MBEDTLS_ECDH_VARIANT_EVEREST /*!< Everest implementation */ -#endif -} mbedtls_ecdh_variant; - -/** - * The context used by the default ECDH implementation. - * - * Later versions might change the structure of this context, therefore users - * should not make any assumptions about the structure of - * mbedtls_ecdh_context_mbed. - */ -typedef struct mbedtls_ecdh_context_mbed { - mbedtls_ecp_group MBEDTLS_PRIVATE(grp); /*!< The elliptic curve used. */ - mbedtls_mpi MBEDTLS_PRIVATE(d); /*!< The private key. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Q); /*!< The public key. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Qp); /*!< The value of the public key of the peer. */ - mbedtls_mpi MBEDTLS_PRIVATE(z); /*!< The shared secret. */ -#if defined(MBEDTLS_ECP_RESTARTABLE) - mbedtls_ecp_restart_ctx MBEDTLS_PRIVATE(rs); /*!< The restart context for EC computations. */ -#endif -} mbedtls_ecdh_context_mbed; -#endif - -/** - * - * \warning Performing multiple operations concurrently on the same - * ECDSA context is not supported; objects of this type - * should not be shared between multiple threads. - * \brief The ECDH context structure. - */ -typedef struct mbedtls_ecdh_context { -#if defined(MBEDTLS_ECDH_LEGACY_CONTEXT) - mbedtls_ecp_group MBEDTLS_PRIVATE(grp); /*!< The elliptic curve used. */ - mbedtls_mpi MBEDTLS_PRIVATE(d); /*!< The private key. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Q); /*!< The public key. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Qp); /*!< The value of the public key of the peer. */ - mbedtls_mpi MBEDTLS_PRIVATE(z); /*!< The shared secret. */ - int MBEDTLS_PRIVATE(point_format); /*!< The format of point export in TLS messages. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Vi); /*!< The blinding value. */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Vf); /*!< The unblinding value. */ - mbedtls_mpi MBEDTLS_PRIVATE(_d); /*!< The previous \p d. */ -#if defined(MBEDTLS_ECP_RESTARTABLE) - int MBEDTLS_PRIVATE(restart_enabled); /*!< The flag for restartable mode. */ - mbedtls_ecp_restart_ctx MBEDTLS_PRIVATE(rs); /*!< The restart context for EC computations. */ -#endif /* MBEDTLS_ECP_RESTARTABLE */ -#else - uint8_t MBEDTLS_PRIVATE(point_format); /*!< The format of point export in TLS messages - as defined in RFC 4492. */ - mbedtls_ecp_group_id MBEDTLS_PRIVATE(grp_id);/*!< The elliptic curve used. */ - mbedtls_ecdh_variant MBEDTLS_PRIVATE(var); /*!< The ECDH implementation/structure used. */ - union { - mbedtls_ecdh_context_mbed MBEDTLS_PRIVATE(mbed_ecdh); -#if defined(MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED) - mbedtls_ecdh_context_everest MBEDTLS_PRIVATE(everest_ecdh); -#endif - } MBEDTLS_PRIVATE(ctx); /*!< Implementation-specific context. The - context in use is specified by the \c var - field. */ -#if defined(MBEDTLS_ECP_RESTARTABLE) - uint8_t MBEDTLS_PRIVATE(restart_enabled); /*!< The flag for restartable mode. Functions of - an alternative implementation not supporting - restartable mode must return - MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED error - if this flag is set. */ -#endif /* MBEDTLS_ECP_RESTARTABLE */ -#endif /* MBEDTLS_ECDH_LEGACY_CONTEXT */ -} -mbedtls_ecdh_context; - -/** - * \brief Return the ECP group for provided context. - * - * \note To access group specific fields, users should use - * `mbedtls_ecp_curve_info_from_grp_id` or - * `mbedtls_ecp_group_load` on the extracted `group_id`. - * - * \param ctx The ECDH context to parse. This must not be \c NULL. - * - * \return The \c mbedtls_ecp_group_id of the context. - */ -mbedtls_ecp_group_id mbedtls_ecdh_get_grp_id(mbedtls_ecdh_context *ctx); - -/** - * \brief Check whether a given group can be used for ECDH. - * - * \param gid The ECP group ID to check. - * - * \return \c 1 if the group can be used, \c 0 otherwise - */ -int mbedtls_ecdh_can_do(mbedtls_ecp_group_id gid); - -/** - * \brief This function generates an ECDH keypair on an elliptic - * curve. - * - * This function performs the first of two core computations - * implemented during the ECDH key exchange. The second core - * computation is performed by mbedtls_ecdh_compute_shared(). - * - * \see ecp.h - * - * \param grp The ECP group to use. This must be initialized and have - * domain parameters loaded, for example through - * mbedtls_ecp_load() or mbedtls_ecp_tls_read_group(). - * \param d The destination MPI (private key). - * This must be initialized. - * \param Q The destination point (public key). - * This must be initialized. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL in case \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return Another \c MBEDTLS_ERR_ECP_XXX or - * \c MBEDTLS_MPI_XXX error code on failure. - */ -int mbedtls_ecdh_gen_public(mbedtls_ecp_group *grp, mbedtls_mpi *d, mbedtls_ecp_point *Q, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function computes the shared secret. - * - * This function performs the second of two core computations - * implemented during the ECDH key exchange. The first core - * computation is performed by mbedtls_ecdh_gen_public(). - * - * \see ecp.h - * - * \note If \p f_rng is not NULL, it is used to implement - * countermeasures against side-channel attacks. - * For more information, see mbedtls_ecp_mul(). - * - * \param grp The ECP group to use. This must be initialized and have - * domain parameters loaded, for example through - * mbedtls_ecp_load() or mbedtls_ecp_tls_read_group(). - * \param z The destination MPI (shared secret). - * This must be initialized. - * \param Q The public key from another party. - * This must be initialized. - * \param d Our secret exponent (private key). - * This must be initialized. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng is \c NULL or doesn't need a - * context argument. - * - * \return \c 0 on success. - * \return Another \c MBEDTLS_ERR_ECP_XXX or - * \c MBEDTLS_MPI_XXX error code on failure. - */ -int mbedtls_ecdh_compute_shared(mbedtls_ecp_group *grp, mbedtls_mpi *z, - const mbedtls_ecp_point *Q, const mbedtls_mpi *d, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function initializes an ECDH context. - * - * \param ctx The ECDH context to initialize. This must not be \c NULL. - */ -void mbedtls_ecdh_init(mbedtls_ecdh_context *ctx); - -/** - * \brief This function sets up the ECDH context with the information - * given. - * - * This function should be called after mbedtls_ecdh_init() but - * before mbedtls_ecdh_make_params(). There is no need to call - * this function before mbedtls_ecdh_read_params(). - * - * This is the first function used by a TLS server for ECDHE - * ciphersuites. - * - * \param ctx The ECDH context to set up. This must be initialized. - * \param grp_id The group id of the group to set up the context for. - * - * \return \c 0 on success. - */ -int mbedtls_ecdh_setup(mbedtls_ecdh_context *ctx, - mbedtls_ecp_group_id grp_id); - -/** - * \brief This function frees a context. - * - * \param ctx The context to free. This may be \c NULL, in which - * case this function does nothing. If it is not \c NULL, - * it must point to an initialized ECDH context. - */ -void mbedtls_ecdh_free(mbedtls_ecdh_context *ctx); - -/** - * \brief This function generates an EC key pair and exports its - * in the format used in a TLS ServerKeyExchange handshake - * message. - * - * This is the second function used by a TLS server for ECDHE - * ciphersuites. (It is called after mbedtls_ecdh_setup().) - * - * \see ecp.h - * - * \param ctx The ECDH context to use. This must be initialized - * and bound to a group, for example via mbedtls_ecdh_setup(). - * \param olen The address at which to store the number of Bytes written. - * \param buf The destination buffer. This must be a writable buffer of - * length \p blen Bytes. - * \param blen The length of the destination buffer \p buf in Bytes. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL in case \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX error code on failure. - */ -int mbedtls_ecdh_make_params(mbedtls_ecdh_context *ctx, size_t *olen, - unsigned char *buf, size_t blen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function parses the ECDHE parameters in a - * TLS ServerKeyExchange handshake message. - * - * \note In a TLS handshake, this is the how the client - * sets up its ECDHE context from the server's public - * ECDHE key material. - * - * \see ecp.h - * - * \param ctx The ECDHE context to use. This must be initialized. - * \param buf On input, \c *buf must be the start of the input buffer. - * On output, \c *buf is updated to point to the end of the - * data that has been read. On success, this is the first byte - * past the end of the ServerKeyExchange parameters. - * On error, this is the point at which an error has been - * detected, which is usually not useful except to debug - * failures. - * \param end The end of the input buffer. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX error code on failure. - * - */ -int mbedtls_ecdh_read_params(mbedtls_ecdh_context *ctx, - const unsigned char **buf, - const unsigned char *end); - -/** - * \brief This function sets up an ECDH context from an EC key. - * - * It is used by clients and servers in place of the - * ServerKeyExchange for static ECDH, and imports ECDH - * parameters from the EC key information of a certificate. - * - * \see ecp.h - * - * \param ctx The ECDH context to set up. This must be initialized. - * \param key The EC key to use. This must be initialized. - * \param side Defines the source of the key. Possible values are: - * - #MBEDTLS_ECDH_OURS: The key is ours. - * - #MBEDTLS_ECDH_THEIRS: The key is that of the peer. - * - * \return \c 0 on success. - * \return Another \c MBEDTLS_ERR_ECP_XXX error code on failure. - * - */ -int mbedtls_ecdh_get_params(mbedtls_ecdh_context *ctx, - const mbedtls_ecp_keypair *key, - mbedtls_ecdh_side side); - -/** - * \brief This function generates a public key and exports it - * as a TLS ClientKeyExchange payload. - * - * This is the second function used by a TLS client for ECDH(E) - * ciphersuites. - * - * \see ecp.h - * - * \param ctx The ECDH context to use. This must be initialized - * and bound to a group, the latter usually by - * mbedtls_ecdh_read_params(). - * \param olen The address at which to store the number of Bytes written. - * This must not be \c NULL. - * \param buf The destination buffer. This must be a writable buffer - * of length \p blen Bytes. - * \param blen The size of the destination buffer \p buf in Bytes. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL in case \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX error code on failure. - */ -int mbedtls_ecdh_make_public(mbedtls_ecdh_context *ctx, size_t *olen, - unsigned char *buf, size_t blen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function parses and processes the ECDHE payload of a - * TLS ClientKeyExchange message. - * - * This is the third function used by a TLS server for ECDH(E) - * ciphersuites. (It is called after mbedtls_ecdh_setup() and - * mbedtls_ecdh_make_params().) - * - * \see ecp.h - * - * \param ctx The ECDH context to use. This must be initialized - * and bound to a group, for example via mbedtls_ecdh_setup(). - * \param buf The pointer to the ClientKeyExchange payload. This must - * be a readable buffer of length \p blen Bytes. - * \param blen The length of the input buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX error code on failure. - */ -int mbedtls_ecdh_read_public(mbedtls_ecdh_context *ctx, - const unsigned char *buf, size_t blen); - -/** - * \brief This function derives and exports the shared secret. - * - * This is the last function used by both TLS client - * and servers. - * - * \note If \p f_rng is not NULL, it is used to implement - * countermeasures against side-channel attacks. - * For more information, see mbedtls_ecp_mul(). - * - * \see ecp.h - - * \param ctx The ECDH context to use. This must be initialized - * and have its own private key generated and the peer's - * public key imported. - * \param olen The address at which to store the total number of - * Bytes written on success. This must not be \c NULL. - * \param buf The buffer to write the generated shared key to. This - * must be a writable buffer of size \p blen Bytes. - * \param blen The length of the destination buffer \p buf in Bytes. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context. This may be \c NULL if \p f_rng - * doesn't need a context argument. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX error code on failure. - */ -int mbedtls_ecdh_calc_secret(mbedtls_ecdh_context *ctx, size_t *olen, - unsigned char *buf, size_t blen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -#if defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief This function enables restartable EC computations for this - * context. (Default: disabled.) - * - * \see \c mbedtls_ecp_set_max_ops() - * - * \note It is not possible to safely disable restartable - * computations once enabled, except by free-ing the context, - * which cancels possible in-progress operations. - * - * \param ctx The ECDH context to use. This must be initialized. - */ -void mbedtls_ecdh_enable_restart(mbedtls_ecdh_context *ctx); -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -#ifdef __cplusplus -} -#endif - -#endif /* ecdh.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ecdsa.h b/android/mbedtls/arm64-v8a/include/mbedtls/ecdsa.h deleted file mode 100644 index c1616615f..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ecdsa.h +++ /dev/null @@ -1,674 +0,0 @@ -/** - * \file ecdsa.h - * - * \brief This file contains ECDSA definitions and functions. - * - * The Elliptic Curve Digital Signature Algorithm (ECDSA) is defined in - * Standards for Efficient Cryptography Group (SECG): - * SEC1 Elliptic Curve Cryptography. - * The use of ECDSA for TLS is defined in RFC-4492: Elliptic Curve - * Cryptography (ECC) Cipher Suites for Transport Layer Security (TLS). - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_ECDSA_H -#define MBEDTLS_ECDSA_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/ecp.h" -#include "mbedtls/md.h" - -/** - * \brief Maximum ECDSA signature size for a given curve bit size - * - * \param bits Curve size in bits - * \return Maximum signature size in bytes - * - * \note This macro returns a compile-time constant if its argument - * is one. It may evaluate its argument multiple times. - */ -/* - * Ecdsa-Sig-Value ::= SEQUENCE { - * r INTEGER, - * s INTEGER - * } - * - * For each of r and s, the value (V) may include an extra initial "0" bit. - */ -#define MBEDTLS_ECDSA_MAX_SIG_LEN(bits) \ - (/*T,L of SEQUENCE*/ ((bits) >= 61 * 8 ? 3 : 2) + \ - /*T,L of r,s*/ 2 * (((bits) >= 127 * 8 ? 3 : 2) + \ - /*V of r,s*/ ((bits) + 8) / 8)) - -/** The maximal size of an ECDSA signature in Bytes. */ -#define MBEDTLS_ECDSA_MAX_LEN MBEDTLS_ECDSA_MAX_SIG_LEN(MBEDTLS_ECP_MAX_BITS) - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief The ECDSA context structure. - * - * \warning Performing multiple operations concurrently on the same - * ECDSA context is not supported; objects of this type - * should not be shared between multiple threads. - * - * \note pk_wrap module assumes that "ecdsa_context" is identical - * to "ecp_keypair" (see for example structure - * "mbedtls_eckey_info" where ECDSA sign/verify functions - * are used also for EC key) - */ -typedef mbedtls_ecp_keypair mbedtls_ecdsa_context; - -#if defined(MBEDTLS_ECP_RESTARTABLE) - -/** - * \brief Internal restart context for ecdsa_verify() - * - * \note Opaque struct, defined in ecdsa.c - */ -typedef struct mbedtls_ecdsa_restart_ver mbedtls_ecdsa_restart_ver_ctx; - -/** - * \brief Internal restart context for ecdsa_sign() - * - * \note Opaque struct, defined in ecdsa.c - */ -typedef struct mbedtls_ecdsa_restart_sig mbedtls_ecdsa_restart_sig_ctx; - -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) -/** - * \brief Internal restart context for ecdsa_sign_det() - * - * \note Opaque struct, defined in ecdsa.c - */ -typedef struct mbedtls_ecdsa_restart_det mbedtls_ecdsa_restart_det_ctx; -#endif - -/** - * \brief General context for resuming ECDSA operations - */ -typedef struct { - mbedtls_ecp_restart_ctx MBEDTLS_PRIVATE(ecp); /*!< base context for ECP restart and - shared administrative info */ - mbedtls_ecdsa_restart_ver_ctx *MBEDTLS_PRIVATE(ver); /*!< ecdsa_verify() sub-context */ - mbedtls_ecdsa_restart_sig_ctx *MBEDTLS_PRIVATE(sig); /*!< ecdsa_sign() sub-context */ -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) - mbedtls_ecdsa_restart_det_ctx *MBEDTLS_PRIVATE(det); /*!< ecdsa_sign_det() sub-context */ -#endif -} mbedtls_ecdsa_restart_ctx; - -#else /* MBEDTLS_ECP_RESTARTABLE */ - -/* Now we can declare functions that take a pointer to that */ -typedef void mbedtls_ecdsa_restart_ctx; - -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -/** - * \brief This function checks whether a given group can be used - * for ECDSA. - * - * \param gid The ECP group ID to check. - * - * \return \c 1 if the group can be used, \c 0 otherwise - */ -int mbedtls_ecdsa_can_do(mbedtls_ecp_group_id gid); - -/** - * \brief This function computes the ECDSA signature of a - * previously-hashed message. - * - * \note The deterministic version implemented in - * mbedtls_ecdsa_sign_det_ext() is usually preferred. - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated - * as defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.3, step 5. - * - * \see ecp.h - * - * \param grp The context for the elliptic curve to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param r The MPI context in which to store the first part - * the signature. This must be initialized. - * \param s The MPI context in which to store the second part - * the signature. This must be initialized. - * \param d The private signing key. This must be initialized. - * \param buf The content to be signed. This is usually the hash of - * the original data to be signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param f_rng The RNG function, used both to generate the ECDSA nonce - * and for blinding. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context parameter. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX - * or \c MBEDTLS_MPI_XXX error code on failure. - */ -int mbedtls_ecdsa_sign(mbedtls_ecp_group *grp, mbedtls_mpi *r, mbedtls_mpi *s, - const mbedtls_mpi *d, const unsigned char *buf, size_t blen, - mbedtls_f_rng_t *f_rng, void *p_rng); - -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) -/** - * \brief This function computes the ECDSA signature of a - * previously-hashed message, deterministic version. - * - * For more information, see RFC-6979: Deterministic - * Usage of the Digital Signature Algorithm (DSA) and Elliptic - * Curve Digital Signature Algorithm (ECDSA). - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated as - * defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.3, step 5. - * - * \see ecp.h - * - * \param grp The context for the elliptic curve to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param r The MPI context in which to store the first part - * the signature. This must be initialized. - * \param s The MPI context in which to store the second part - * the signature. This must be initialized. - * \param d The private signing key. This must be initialized - * and setup, for example through mbedtls_ecp_gen_privkey(). - * \param buf The hashed content to be signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param md_alg The hash algorithm used to hash the original data. - * \param f_rng_blind The RNG function used for blinding. This must not be - * \c NULL. - * \param p_rng_blind The RNG context to be passed to \p f_rng_blind. This - * may be \c NULL if \p f_rng_blind doesn't need a context - * parameter. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX - * error code on failure. - */ -int mbedtls_ecdsa_sign_det_ext(mbedtls_ecp_group *grp, mbedtls_mpi *r, - mbedtls_mpi *s, const mbedtls_mpi *d, - const unsigned char *buf, size_t blen, - mbedtls_md_type_t md_alg, - mbedtls_f_rng_t *f_rng_blind, - void *p_rng_blind); -#endif /* MBEDTLS_ECDSA_DETERMINISTIC */ - -#if !defined(MBEDTLS_ECDSA_SIGN_ALT) -/** - * \brief This function computes the ECDSA signature of a - * previously-hashed message, in a restartable way. - * - * \note The deterministic version implemented in - * mbedtls_ecdsa_sign_det_restartable() is usually - * preferred. - * - * \note This function is like \c mbedtls_ecdsa_sign() but - * it can return early and restart according to the - * limit set with \c mbedtls_ecp_set_max_ops() to - * reduce blocking. - * - * \note If the bitlength of the message hash is larger - * than the bitlength of the group order, then the - * hash is truncated as defined in Standards for - * Efficient Cryptography Group (SECG): SEC1 Elliptic - * Curve Cryptography, section 4.1.3, step 5. - * - * \see ecp.h - * - * \param grp The context for the elliptic curve to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param r The MPI context in which to store the first part - * the signature. This must be initialized. - * \param s The MPI context in which to store the second part - * the signature. This must be initialized. - * \param d The private signing key. This must be initialized - * and setup, for example through - * mbedtls_ecp_gen_privkey(). - * \param buf The hashed content to be signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param f_rng The RNG function used to generate the ECDSA nonce. - * This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context parameter. - * \param f_rng_blind The RNG function used for blinding. This must not be - * \c NULL. - * \param p_rng_blind The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context parameter. - * \param rs_ctx The restart context to use. This may be \c NULL - * to disable restarting. If it is not \c NULL, it - * must point to an initialized restart context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c - * mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX, \c - * MBEDTLS_ERR_MPI_XXX or \c MBEDTLS_ERR_ASN1_XXX - * error code on failure. - */ -int mbedtls_ecdsa_sign_restartable( - mbedtls_ecp_group *grp, - mbedtls_mpi *r, mbedtls_mpi *s, - const mbedtls_mpi *d, - const unsigned char *buf, size_t blen, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_f_rng_t *f_rng_blind, - void *p_rng_blind, - mbedtls_ecdsa_restart_ctx *rs_ctx); - -#endif /* !MBEDTLS_ECDSA_SIGN_ALT */ - -#if defined(MBEDTLS_ECDSA_DETERMINISTIC) - -/** - * \brief This function computes the ECDSA signature of a - * previously-hashed message, in a restartable way. - * - * \note This function is like \c - * mbedtls_ecdsa_sign_det_ext() but it can return - * early and restart according to the limit set with - * \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \note If the bitlength of the message hash is larger - * than the bitlength of the group order, then the - * hash is truncated as defined in Standards for - * Efficient Cryptography Group (SECG): SEC1 Elliptic - * Curve Cryptography, section 4.1.3, step 5. - * - * \see ecp.h - * - * \param grp The context for the elliptic curve to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param r The MPI context in which to store the first part - * the signature. This must be initialized. - * \param s The MPI context in which to store the second part - * the signature. This must be initialized. - * \param d The private signing key. This must be initialized - * and setup, for example through - * mbedtls_ecp_gen_privkey(). - * \param buf The hashed content to be signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param md_alg The hash algorithm used to hash the original data. - * \param f_rng_blind The RNG function used for blinding. This must not be - * \c NULL. - * \param p_rng_blind The RNG context to be passed to \p f_rng_blind. This may be - * \c NULL if \p f_rng_blind doesn't need a context parameter. - * \param rs_ctx The restart context to use. This may be \c NULL - * to disable restarting. If it is not \c NULL, it - * must point to an initialized restart context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c - * mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX, \c - * MBEDTLS_ERR_MPI_XXX or \c MBEDTLS_ERR_ASN1_XXX - * error code on failure. - */ -int mbedtls_ecdsa_sign_det_restartable( - mbedtls_ecp_group *grp, - mbedtls_mpi *r, mbedtls_mpi *s, - const mbedtls_mpi *d, const unsigned char *buf, size_t blen, - mbedtls_md_type_t md_alg, - mbedtls_f_rng_t *f_rng_blind, - void *p_rng_blind, - mbedtls_ecdsa_restart_ctx *rs_ctx); - -#endif /* MBEDTLS_ECDSA_DETERMINISTIC */ - -/** - * \brief This function verifies the ECDSA signature of a - * previously-hashed message. - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated as - * defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.4, step 3. - * - * \see ecp.h - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param buf The hashed content that was signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param Q The public key to use for verification. This must be - * initialized and setup. - * \param r The first integer of the signature. - * This must be initialized. - * \param s The second integer of the signature. - * This must be initialized. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX - * error code on failure. - */ -int mbedtls_ecdsa_verify(mbedtls_ecp_group *grp, - const unsigned char *buf, size_t blen, - const mbedtls_ecp_point *Q, const mbedtls_mpi *r, - const mbedtls_mpi *s); - -#if !defined(MBEDTLS_ECDSA_VERIFY_ALT) -/** - * \brief This function verifies the ECDSA signature of a - * previously-hashed message, in a restartable manner - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated as - * defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.4, step 3. - * - * \see ecp.h - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param buf The hashed content that was signed. This must be a readable - * buffer of length \p blen Bytes. It may be \c NULL if - * \p blen is zero. - * \param blen The length of \p buf in Bytes. - * \param Q The public key to use for verification. This must be - * initialized and setup. - * \param r The first integer of the signature. - * This must be initialized. - * \param s The second integer of the signature. - * This must be initialized. - * \param rs_ctx The restart context to use. This may be \c NULL to disable - * restarting. If it is not \c NULL, it must point to an - * initialized restart context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX - * error code on failure. - */ -int mbedtls_ecdsa_verify_restartable(mbedtls_ecp_group *grp, - const unsigned char *buf, size_t blen, - const mbedtls_ecp_point *Q, - const mbedtls_mpi *r, - const mbedtls_mpi *s, - mbedtls_ecdsa_restart_ctx *rs_ctx); - -#endif /* !MBEDTLS_ECDSA_VERIFY_ALT */ - -/** - * \brief This function computes the ECDSA signature and writes it - * to a buffer, serialized as defined in RFC-4492: - * Elliptic Curve Cryptography (ECC) Cipher Suites for - * Transport Layer Security (TLS). - * - * \warning It is not thread-safe to use the same context in - * multiple threads. - * - * \note The deterministic version is used if - * #MBEDTLS_ECDSA_DETERMINISTIC is defined. For more - * information, see RFC-6979: Deterministic Usage - * of the Digital Signature Algorithm (DSA) and Elliptic - * Curve Digital Signature Algorithm (ECDSA). - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated as - * defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.3, step 5. - * - * \see ecp.h - * - * \param ctx The ECDSA context to use. This must be initialized - * and have a group and private key bound to it, for example - * via mbedtls_ecdsa_genkey() or mbedtls_ecdsa_from_keypair(). - * \param md_alg The message digest that was used to hash the message. - * \param hash The message hash to be signed. This must be a readable - * buffer of length \p hlen Bytes. - * \param hlen The length of the hash \p hash in Bytes. - * \param sig The buffer to which to write the signature. This must be a - * writable buffer of length at least twice as large as the - * size of the curve used, plus 9. For example, 73 Bytes if - * a 256-bit curve is used. A buffer length of - * #MBEDTLS_ECDSA_MAX_LEN is always safe. - * \param sig_size The size of the \p sig buffer in bytes. - * \param slen The address at which to store the actual length of - * the signature written. Must not be \c NULL. - * \param f_rng The RNG function. This is used for blinding. - * If #MBEDTLS_ECDSA_DETERMINISTIC is unset, this is also - * used to generate the ECDSA nonce. - * This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng is \c NULL or doesn't use a context. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX, \c MBEDTLS_ERR_MPI_XXX or - * \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_ecdsa_write_signature(mbedtls_ecdsa_context *ctx, - mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hlen, - unsigned char *sig, size_t sig_size, size_t *slen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function computes the ECDSA signature and writes it - * to a buffer, in a restartable way. - * - * \see \c mbedtls_ecdsa_write_signature() - * - * \note This function is like \c mbedtls_ecdsa_write_signature() - * but it can return early and restart according to the limit - * set with \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \param ctx The ECDSA context to use. This must be initialized - * and have a group and private key bound to it, for example - * via mbedtls_ecdsa_genkey() or mbedtls_ecdsa_from_keypair(). - * \param md_alg The message digest that was used to hash the message. - * \param hash The message hash to be signed. This must be a readable - * buffer of length \p hlen Bytes. - * \param hlen The length of the hash \p hash in Bytes. - * \param sig The buffer to which to write the signature. This must be a - * writable buffer of length at least twice as large as the - * size of the curve used, plus 9. For example, 73 Bytes if - * a 256-bit curve is used. A buffer length of - * #MBEDTLS_ECDSA_MAX_LEN is always safe. - * \param sig_size The size of the \p sig buffer in bytes. - * \param slen The address at which to store the actual length of - * the signature written. Must not be \c NULL. - * \param f_rng The RNG function. This is used for blinding. - * If #MBEDTLS_ECDSA_DETERMINISTIC is unset, this is also - * used to generate the ECDSA nonce. - * This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng is \c NULL or doesn't use a context. - * \param rs_ctx The restart context to use. This may be \c NULL to disable - * restarting. If it is not \c NULL, it must point to an - * initialized restart context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX, \c MBEDTLS_ERR_MPI_XXX or - * \c MBEDTLS_ERR_ASN1_XXX error code on failure. - */ -int mbedtls_ecdsa_write_signature_restartable(mbedtls_ecdsa_context *ctx, - mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hlen, - unsigned char *sig, size_t sig_size, size_t *slen, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_ecdsa_restart_ctx *rs_ctx); - -/** - * \brief This function reads and verifies an ECDSA signature. - * - * \note If the bitlength of the message hash is larger than the - * bitlength of the group order, then the hash is truncated as - * defined in Standards for Efficient Cryptography Group - * (SECG): SEC1 Elliptic Curve Cryptography, section - * 4.1.4, step 3. - * - * \see ecp.h - * - * \param ctx The ECDSA context to use. This must be initialized - * and have a group and public key bound to it. - * \param hash The message hash that was signed. This must be a readable - * buffer of length \p hlen Bytes. - * \param hlen The size of the hash \p hash. - * \param sig The signature to read and verify. This must be a readable - * buffer of length \p slen Bytes. - * \param slen The size of \p sig in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if signature is invalid. - * \return #MBEDTLS_ERR_ECP_SIG_LEN_MISMATCH if there is a valid - * signature in \p sig, but its length is less than \p siglen. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_ERR_MPI_XXX - * error code on failure for any other reason. - */ -int mbedtls_ecdsa_read_signature(mbedtls_ecdsa_context *ctx, - const unsigned char *hash, size_t hlen, - const unsigned char *sig, size_t slen); - -/** - * \brief This function reads and verifies an ECDSA signature, - * in a restartable way. - * - * \see \c mbedtls_ecdsa_read_signature() - * - * \note This function is like \c mbedtls_ecdsa_read_signature() - * but it can return early and restart according to the limit - * set with \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \param ctx The ECDSA context to use. This must be initialized - * and have a group and public key bound to it. - * \param hash The message hash that was signed. This must be a readable - * buffer of length \p hlen Bytes. - * \param hlen The size of the hash \p hash. - * \param sig The signature to read and verify. This must be a readable - * buffer of length \p slen Bytes. - * \param slen The size of \p sig in Bytes. - * \param rs_ctx The restart context to use. This may be \c NULL to disable - * restarting. If it is not \c NULL, it must point to an - * initialized restart context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if signature is invalid. - * \return #MBEDTLS_ERR_ECP_SIG_LEN_MISMATCH if there is a valid - * signature in \p sig, but its length is less than \p siglen. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_ERR_MPI_XXX - * error code on failure for any other reason. - */ -int mbedtls_ecdsa_read_signature_restartable(mbedtls_ecdsa_context *ctx, - const unsigned char *hash, size_t hlen, - const unsigned char *sig, size_t slen, - mbedtls_ecdsa_restart_ctx *rs_ctx); - -/** - * \brief This function generates an ECDSA keypair on the given curve. - * - * \see ecp.h - * - * \param ctx The ECDSA context to store the keypair in. - * This must be initialized. - * \param gid The elliptic curve to use. One of the various - * \c MBEDTLS_ECP_DP_XXX macros depending on configuration. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX code on failure. - */ -int mbedtls_ecdsa_genkey(mbedtls_ecdsa_context *ctx, mbedtls_ecp_group_id gid, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** - * \brief This function sets up an ECDSA context from an EC key pair. - * - * \see ecp.h - * - * \param ctx The ECDSA context to setup. This must be initialized. - * \param key The EC key to use. This must be initialized and hold - * a private-public key pair or a public key. In the former - * case, the ECDSA context may be used for signature creation - * and verification after this call. In the latter case, it - * may be used for signature verification. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX code on failure. - */ -int mbedtls_ecdsa_from_keypair(mbedtls_ecdsa_context *ctx, - const mbedtls_ecp_keypair *key); - -/** - * \brief This function initializes an ECDSA context. - * - * \param ctx The ECDSA context to initialize. - * This must not be \c NULL. - */ -void mbedtls_ecdsa_init(mbedtls_ecdsa_context *ctx); - -/** - * \brief This function frees an ECDSA context. - * - * \param ctx The ECDSA context to free. This may be \c NULL, - * in which case this function does nothing. If it - * is not \c NULL, it must be initialized. - */ -void mbedtls_ecdsa_free(mbedtls_ecdsa_context *ctx); - -#if defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Initialize a restart context. - * - * \param ctx The restart context to initialize. - * This must not be \c NULL. - */ -void mbedtls_ecdsa_restart_init(mbedtls_ecdsa_restart_ctx *ctx); - -/** - * \brief Free the components of a restart context. - * - * \param ctx The restart context to free. This may be \c NULL, - * in which case this function does nothing. If it - * is not \c NULL, it must be initialized. - */ -void mbedtls_ecdsa_restart_free(mbedtls_ecdsa_restart_ctx *ctx); -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -#ifdef __cplusplus -} -#endif - -#endif /* ecdsa.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ecjpake.h b/android/mbedtls/arm64-v8a/include/mbedtls/ecjpake.h deleted file mode 100644 index 7da8cb4e5..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ecjpake.h +++ /dev/null @@ -1,298 +0,0 @@ -/** - * \file ecjpake.h - * - * \brief Elliptic curve J-PAKE - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_ECJPAKE_H -#define MBEDTLS_ECJPAKE_H -#include "mbedtls/private_access.h" - -/* - * J-PAKE is a password-authenticated key exchange that allows deriving a - * strong shared secret from a (potentially low entropy) pre-shared - * passphrase, with forward secrecy and mutual authentication. - * https://en.wikipedia.org/wiki/Password_Authenticated_Key_Exchange_by_Juggling - * - * This file implements the Elliptic Curve variant of J-PAKE, - * as defined in Chapter 7.4 of the Thread v1.0 Specification, - * available to members of the Thread Group http://threadgroup.org/ - * - * As the J-PAKE algorithm is inherently symmetric, so is our API. - * Each party needs to send its first round message, in any order, to the - * other party, then each sends its second round message, in any order. - * The payloads are serialized in a way suitable for use in TLS, but could - * also be use outside TLS. - */ -#include "mbedtls/build_info.h" - -#include "mbedtls/ecp.h" -#include "mbedtls/md.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Roles in the EC J-PAKE exchange - */ -typedef enum { - MBEDTLS_ECJPAKE_CLIENT = 0, /**< Client */ - MBEDTLS_ECJPAKE_SERVER, /**< Server */ - MBEDTLS_ECJPAKE_NONE, /**< Undefined */ -} mbedtls_ecjpake_role; - -#if !defined(MBEDTLS_ECJPAKE_ALT) -/** - * EC J-PAKE context structure. - * - * J-PAKE is a symmetric protocol, except for the identifiers used in - * Zero-Knowledge Proofs, and the serialization of the second message - * (KeyExchange) as defined by the Thread spec. - * - * In order to benefit from this symmetry, we choose a different naming - * convention from the Thread v1.0 spec. Correspondence is indicated in the - * description as a pair C: client name, S: server name - */ -typedef struct mbedtls_ecjpake_context { - mbedtls_md_type_t MBEDTLS_PRIVATE(md_type); /**< Hash to use */ - mbedtls_ecp_group MBEDTLS_PRIVATE(grp); /**< Elliptic curve */ - mbedtls_ecjpake_role MBEDTLS_PRIVATE(role); /**< Are we client or server? */ - int MBEDTLS_PRIVATE(point_format); /**< Format for point export */ - - mbedtls_ecp_point MBEDTLS_PRIVATE(Xm1); /**< My public key 1 C: X1, S: X3 */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Xm2); /**< My public key 2 C: X2, S: X4 */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Xp1); /**< Peer public key 1 C: X3, S: X1 */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Xp2); /**< Peer public key 2 C: X4, S: X2 */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Xp); /**< Peer public key C: Xs, S: Xc */ - - mbedtls_mpi MBEDTLS_PRIVATE(xm1); /**< My private key 1 C: x1, S: x3 */ - mbedtls_mpi MBEDTLS_PRIVATE(xm2); /**< My private key 2 C: x2, S: x4 */ - - mbedtls_mpi MBEDTLS_PRIVATE(s); /**< Pre-shared secret (passphrase) */ -} mbedtls_ecjpake_context; - -#else /* MBEDTLS_ECJPAKE_ALT */ -#include "ecjpake_alt.h" -#endif /* MBEDTLS_ECJPAKE_ALT */ - -/** - * \brief Initialize an ECJPAKE context. - * - * \param ctx The ECJPAKE context to initialize. - * This must not be \c NULL. - */ -void mbedtls_ecjpake_init(mbedtls_ecjpake_context *ctx); - -/** - * \brief Set up an ECJPAKE context for use. - * - * \note Currently the only values for hash/curve allowed by the - * standard are #MBEDTLS_MD_SHA256/#MBEDTLS_ECP_DP_SECP256R1. - * - * \param ctx The ECJPAKE context to set up. This must be initialized. - * \param role The role of the caller. This must be either - * #MBEDTLS_ECJPAKE_CLIENT or #MBEDTLS_ECJPAKE_SERVER. - * \param hash The identifier of the hash function to use, - * for example #MBEDTLS_MD_SHA256. - * \param curve The identifier of the elliptic curve to use, - * for example #MBEDTLS_ECP_DP_SECP256R1. - * \param secret The pre-shared secret (passphrase). This must be - * a readable not empty buffer of length \p len Bytes. It need - * only be valid for the duration of this call. - * \param len The length of the pre-shared secret \p secret. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_setup(mbedtls_ecjpake_context *ctx, - mbedtls_ecjpake_role role, - mbedtls_md_type_t hash, - mbedtls_ecp_group_id curve, - const unsigned char *secret, - size_t len); - -/** - * \brief Set the point format for future reads and writes. - * - * \param ctx The ECJPAKE context to configure. - * \param point_format The point format to use: - * #MBEDTLS_ECP_PF_UNCOMPRESSED (default) - * or #MBEDTLS_ECP_PF_COMPRESSED. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if \p point_format - * is invalid. - */ -int mbedtls_ecjpake_set_point_format(mbedtls_ecjpake_context *ctx, - int point_format); - -/** - * \brief Check if an ECJPAKE context is ready for use. - * - * \param ctx The ECJPAKE context to check. This must be - * initialized. - * - * \return \c 0 if the context is ready for use. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA otherwise. - */ -int mbedtls_ecjpake_check(const mbedtls_ecjpake_context *ctx); - -/** - * \brief Generate and write the first round message - * (TLS: contents of the Client/ServerHello extension, - * excluding extension type and length bytes). - * - * \param ctx The ECJPAKE context to use. This must be - * initialized and set up. - * \param buf The buffer to write the contents to. This must be a - * writable buffer of length \p len Bytes. - * \param len The length of \p buf in Bytes. - * \param olen The address at which to store the total number - * of Bytes written to \p buf. This must not be \c NULL. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This - * may be \c NULL if \p f_rng doesn't use a context. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_write_round_one(mbedtls_ecjpake_context *ctx, - unsigned char *buf, size_t len, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Read and process the first round message - * (TLS: contents of the Client/ServerHello extension, - * excluding extension type and length bytes). - * - * \param ctx The ECJPAKE context to use. This must be initialized - * and set up. - * \param buf The buffer holding the first round message. This must - * be a readable buffer of length \p len Bytes. - * \param len The length in Bytes of \p buf. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_read_round_one(mbedtls_ecjpake_context *ctx, - const unsigned char *buf, - size_t len); - -/** - * \brief Generate and write the second round message - * (TLS: contents of the Client/ServerKeyExchange). - * - * \param ctx The ECJPAKE context to use. This must be initialized, - * set up, and already have performed round one. - * \param buf The buffer to write the round two contents to. - * This must be a writable buffer of length \p len Bytes. - * \param len The size of \p buf in Bytes. - * \param olen The address at which to store the total number of Bytes - * written to \p buf. This must not be \c NULL. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This - * may be \c NULL if \p f_rng doesn't use a context. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_write_round_two(mbedtls_ecjpake_context *ctx, - unsigned char *buf, size_t len, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Read and process the second round message - * (TLS: contents of the Client/ServerKeyExchange). - * - * \param ctx The ECJPAKE context to use. This must be initialized - * and set up and already have performed round one. - * \param buf The buffer holding the second round message. This must - * be a readable buffer of length \p len Bytes. - * \param len The length in Bytes of \p buf. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_read_round_two(mbedtls_ecjpake_context *ctx, - const unsigned char *buf, - size_t len); - -/** - * \brief Derive the shared secret - * (TLS: Pre-Master Secret). - * - * \param ctx The ECJPAKE context to use. This must be initialized, - * set up and have performed both round one and two. - * \param buf The buffer to write the derived secret to. This must - * be a writable buffer of length \p len Bytes. - * \param len The length of \p buf in Bytes. - * \param olen The address at which to store the total number of Bytes - * written to \p buf. This must not be \c NULL. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This - * may be \c NULL if \p f_rng doesn't use a context. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_derive_secret(mbedtls_ecjpake_context *ctx, - unsigned char *buf, size_t len, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Write the shared key material to be passed to a Key - * Derivation Function as described in RFC8236. - * - * \param ctx The ECJPAKE context to use. This must be initialized, - * set up and have performed both round one and two. - * \param buf The buffer to write the derived secret to. This must - * be a writable buffer of length \p len Bytes. - * \param len The length of \p buf in Bytes. - * \param olen The address at which to store the total number of bytes - * written to \p buf. This must not be \c NULL. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This - * may be \c NULL if \p f_rng doesn't use a context. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_ecjpake_write_shared_key(mbedtls_ecjpake_context *ctx, - unsigned char *buf, size_t len, size_t *olen, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This clears an ECJPAKE context and frees any - * embedded data structure. - * - * \param ctx The ECJPAKE context to free. This may be \c NULL, - * in which case this function does nothing. If it is not - * \c NULL, it must point to an initialized ECJPAKE context. - */ -void mbedtls_ecjpake_free(mbedtls_ecjpake_context *ctx); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if a test failed - */ -int mbedtls_ecjpake_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - - -#endif /* ecjpake.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ecp.h b/android/mbedtls/arm64-v8a/include/mbedtls/ecp.h deleted file mode 100644 index 5cc027143..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ecp.h +++ /dev/null @@ -1,1528 +0,0 @@ -/** - * \file ecp.h - * - * \brief This file provides an API for Elliptic Curves over GF(P) (ECP). - * - * The use of ECP in cryptography and TLS is defined in - * Standards for Efficient Cryptography Group (SECG): SEC1 - * Elliptic Curve Cryptography and - * RFC-4492: Elliptic Curve Cryptography (ECC) Cipher Suites - * for Transport Layer Security (TLS). - * - * RFC-2409: The Internet Key Exchange (IKE) defines ECP - * group types. - * - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_ECP_H -#define MBEDTLS_ECP_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include "mbedtls/bignum.h" - -/* - * ECP error codes - */ -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_ECP_BAD_INPUT_DATA -0x4F80 -/** The buffer is too small to write to. */ -#define MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL -0x4F00 -/** The requested feature is not available, for example, the requested curve is not supported. */ -#define MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE -0x4E80 -/** The signature is not valid. */ -#define MBEDTLS_ERR_ECP_VERIFY_FAILED -0x4E00 -/** Memory allocation failed. */ -#define MBEDTLS_ERR_ECP_ALLOC_FAILED -0x4D80 -/** Generation of random value, such as ephemeral key, failed. */ -#define MBEDTLS_ERR_ECP_RANDOM_FAILED -0x4D00 -/** Invalid private or public key. */ -#define MBEDTLS_ERR_ECP_INVALID_KEY -0x4C80 -/** The buffer contains a valid signature followed by more data. */ -#define MBEDTLS_ERR_ECP_SIG_LEN_MISMATCH -0x4C00 -/** Operation in progress, call again with the same parameters to continue. */ -#define MBEDTLS_ERR_ECP_IN_PROGRESS -0x4B00 - -/* Flags indicating whether to include code that is specific to certain - * types of curves. These flags are for internal library use only. */ -#if defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) || \ - defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) -#define MBEDTLS_ECP_SHORT_WEIERSTRASS_ENABLED -#endif -#if defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) || \ - defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) -#define MBEDTLS_ECP_MONTGOMERY_ENABLED -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Domain-parameter identifiers: curve, subgroup, and generator. - * - * \note Only curves over prime fields are supported. - * - * \warning This library does not support validation of arbitrary domain - * parameters. Therefore, only standardized domain parameters from trusted - * sources should be used. See mbedtls_ecp_group_load(). - */ -/* Note: when adding a new curve: - * - Add it at the end of this enum, otherwise you'll break the ABI by - * changing the numerical value for existing curves. - * - Increment MBEDTLS_ECP_DP_MAX below if needed. - * - Update the calculation of MBEDTLS_ECP_MAX_BITS below. - * - Add the corresponding MBEDTLS_ECP_DP_xxx_ENABLED macro definition to - * mbedtls_config.h. - * - List the curve as a dependency of MBEDTLS_ECP_C and - * MBEDTLS_ECDSA_C if supported in check_config.h. - * - Add the curve to the appropriate curve type macro - * MBEDTLS_ECP_yyy_ENABLED above. - * - Add the necessary definitions to ecp_curves.c. - * - Add the curve to the ecp_supported_curves array in ecp.c. - * - Add the curve to applicable profiles in x509_crt.c. - * - Add the curve to applicable presets in ssl_tls.c. - */ -typedef enum { - MBEDTLS_ECP_DP_NONE = 0, /*!< Curve not defined. */ - MBEDTLS_ECP_DP_SECP192R1, /*!< Domain parameters for the 192-bit curve defined by FIPS 186-4 and SEC1. */ - MBEDTLS_ECP_DP_SECP224R1, /*!< Domain parameters for the 224-bit curve defined by FIPS 186-4 and SEC1. */ - MBEDTLS_ECP_DP_SECP256R1, /*!< Domain parameters for the 256-bit curve defined by FIPS 186-4 and SEC1. */ - MBEDTLS_ECP_DP_SECP384R1, /*!< Domain parameters for the 384-bit curve defined by FIPS 186-4 and SEC1. */ - MBEDTLS_ECP_DP_SECP521R1, /*!< Domain parameters for the 521-bit curve defined by FIPS 186-4 and SEC1. */ - MBEDTLS_ECP_DP_BP256R1, /*!< Domain parameters for 256-bit Brainpool curve. */ - MBEDTLS_ECP_DP_BP384R1, /*!< Domain parameters for 384-bit Brainpool curve. */ - MBEDTLS_ECP_DP_BP512R1, /*!< Domain parameters for 512-bit Brainpool curve. */ - MBEDTLS_ECP_DP_CURVE25519, /*!< Domain parameters for Curve25519. */ - MBEDTLS_ECP_DP_SECP192K1, /*!< Domain parameters for 192-bit "Koblitz" curve. */ - MBEDTLS_ECP_DP_SECP224K1, /*!< Domain parameters for 224-bit "Koblitz" curve. */ - MBEDTLS_ECP_DP_SECP256K1, /*!< Domain parameters for 256-bit "Koblitz" curve. */ - MBEDTLS_ECP_DP_CURVE448, /*!< Domain parameters for Curve448. */ -} mbedtls_ecp_group_id; - -/** - * The number of supported curves, plus one for #MBEDTLS_ECP_DP_NONE. - */ -#define MBEDTLS_ECP_DP_MAX 14 - -/* - * Curve types - */ -typedef enum { - MBEDTLS_ECP_TYPE_NONE = 0, - MBEDTLS_ECP_TYPE_SHORT_WEIERSTRASS, /* y^2 = x^3 + a x + b */ - MBEDTLS_ECP_TYPE_MONTGOMERY, /* y^2 = x^3 + a x^2 + x */ -} mbedtls_ecp_curve_type; - -/** - * Curve information, for use by other modules. - * - * The fields of this structure are part of the public API and can be - * accessed directly by applications. Future versions of the library may - * add extra fields or reorder existing fields. - */ -typedef struct mbedtls_ecp_curve_info { - mbedtls_ecp_group_id grp_id; /*!< An internal identifier. */ - uint16_t tls_id; /*!< The TLS NamedCurve identifier. */ - uint16_t bit_size; /*!< The curve size in bits. */ - const char *name; /*!< A human-friendly name. */ -} mbedtls_ecp_curve_info; - -/** - * \brief The ECP point structure, in Jacobian coordinates. - * - * \note All functions expect and return points satisfying - * the following condition: Z == 0 or - * Z == 1. Other values of \p Z are - * used only by internal functions. - * The point is zero, or "at infinity", if Z == 0. - * Otherwise, \p X and \p Y are its standard (affine) - * coordinates. - */ -typedef struct mbedtls_ecp_point { - mbedtls_mpi MBEDTLS_PRIVATE(X); /*!< The X coordinate of the ECP point. */ - mbedtls_mpi MBEDTLS_PRIVATE(Y); /*!< The Y coordinate of the ECP point. */ - mbedtls_mpi MBEDTLS_PRIVATE(Z); /*!< The Z coordinate of the ECP point. */ -} -mbedtls_ecp_point; - -#if !defined(MBEDTLS_ECP_ALT) -/* - * default Mbed TLS elliptic curve arithmetic implementation - * - * (in case MBEDTLS_ECP_ALT is defined then the developer has to provide an - * alternative implementation for the whole module and it will replace this - * one.) - */ - -/** - * \brief The ECP group structure. - * - * We consider two types of curve equations: - *
  • Short Weierstrass: y^2 = x^3 + A x + B mod P - * (SEC1 + RFC-4492)
    • - *
  • Montgomery: y^2 = x^3 + A x^2 + x mod P (Curve25519, - * Curve448)
- * In both cases, the generator (\p G) for a prime-order subgroup is fixed. - * - * For Short Weierstrass, this subgroup is the whole curve, and its - * cardinality is denoted by \p N. Our code requires that \p N is an - * odd prime as mbedtls_ecp_mul() requires an odd number, and - * mbedtls_ecdsa_sign() requires that it is prime for blinding purposes. - * - * The default implementation only initializes \p A without setting it to the - * authentic value for curves with A = -3(SECP256R1, etc), in which - * case you need to load \p A by yourself when using domain parameters directly, - * for example: - * \code - * mbedtls_mpi_init(&A); - * mbedtls_ecp_group_init(&grp); - * CHECK_RETURN(mbedtls_ecp_group_load(&grp, grp_id)); - * if (mbedtls_ecp_group_a_is_minus_3(&grp)) { - * CHECK_RETURN(mbedtls_mpi_sub_int(&A, &grp.P, 3)); - * } else { - * CHECK_RETURN(mbedtls_mpi_copy(&A, &grp.A)); - * } - * - * do_something_with_a(&A); - * - * cleanup: - * mbedtls_mpi_free(&A); - * mbedtls_ecp_group_free(&grp); - * \endcode - * - * For Montgomery curves, we do not store \p A, but (A + 2) / 4, - * which is the quantity used in the formulas. Additionally, \p nbits is - * not the size of \p N but the required size for private keys. - * - * If \p modp is NULL, reduction modulo \p P is done using a generic algorithm. - * Otherwise, \p modp must point to a function that takes an \p mbedtls_mpi in the - * range of 0..2^(2*pbits)-1, and transforms it in-place to an integer - * which is congruent mod \p P to the given MPI, and is close enough to \p pbits - * in size, so that it may be efficiently brought in the 0..P-1 range by a few - * additions or subtractions. Therefore, it is only an approximate modular - * reduction. It must return 0 on success and non-zero on failure. - * - * \note Alternative implementations of the ECP module must obey the - * following constraints. - * * Group IDs must be distinct: if two group structures have - * the same ID, then they must be identical. - * * The fields \c id, \c P, \c A, \c B, \c G, \c N, - * \c pbits and \c nbits must have the same type and semantics - * as in the built-in implementation. - * They must be available for reading, but direct modification - * of these fields does not need to be supported. - * They do not need to be at the same offset in the structure. - */ -typedef struct mbedtls_ecp_group { - mbedtls_ecp_group_id id; /*!< An internal group identifier. */ - mbedtls_mpi P; /*!< The prime modulus of the base field. */ - mbedtls_mpi A; /*!< For Short Weierstrass: \p A in the equation. Note that - \p A is not set to the authentic value in some cases. - Refer to detailed description of ::mbedtls_ecp_group if - using domain parameters in the structure. - For Montgomery curves: (A + 2) / 4. */ - mbedtls_mpi B; /*!< For Short Weierstrass: \p B in the equation. - For Montgomery curves: unused. */ - mbedtls_ecp_point G; /*!< The generator of the subgroup used. */ - mbedtls_mpi N; /*!< The order of \p G. */ - size_t pbits; /*!< The number of bits in \p P.*/ - size_t nbits; /*!< For Short Weierstrass: The number of bits in \p P. - For Montgomery curves: the number of bits in the - private keys. */ - /* End of public fields */ - - unsigned int MBEDTLS_PRIVATE(h); /*!< \internal 1 if the constants are static. */ - int(*MBEDTLS_PRIVATE(modp))(mbedtls_mpi *); /*!< The function for fast pseudo-reduction - mod \p P (see above).*/ - int(*MBEDTLS_PRIVATE(t_pre))(mbedtls_ecp_point *, void *); /*!< Unused. */ - int(*MBEDTLS_PRIVATE(t_post))(mbedtls_ecp_point *, void *); /*!< Unused. */ - void *MBEDTLS_PRIVATE(t_data); /*!< Unused. */ - mbedtls_ecp_point *MBEDTLS_PRIVATE(T); /*!< Pre-computed points for ecp_mul_comb(). */ - size_t MBEDTLS_PRIVATE(T_size); /*!< The number of dynamic allocated pre-computed points. */ -} -mbedtls_ecp_group; - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h, or define them using the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_ECP_WINDOW_SIZE) -/* - * Maximum "window" size used for point multiplication. - * Default: a point where higher memory usage yields diminishing performance - * returns. - * Minimum value: 2. Maximum value: 7. - * - * Result is an array of at most ( 1 << ( MBEDTLS_ECP_WINDOW_SIZE - 1 ) ) - * points used for point multiplication. This value is directly tied to EC - * peak memory usage, so decreasing it by one should roughly cut memory usage - * by two (if large curves are in use). - * - * Reduction in size may reduce speed, but larger curves are impacted first. - * Sample performances (in ECDHE handshakes/s, with FIXED_POINT_OPTIM = 1): - * w-size: 6 5 4 3 2 - * 521 145 141 135 120 97 - * 384 214 209 198 177 146 - * 256 320 320 303 262 226 - * 224 475 475 453 398 342 - * 192 640 640 633 587 476 - */ -#define MBEDTLS_ECP_WINDOW_SIZE 4 /**< The maximum window size used. */ -#endif /* MBEDTLS_ECP_WINDOW_SIZE */ - -#if !defined(MBEDTLS_ECP_FIXED_POINT_OPTIM) -/* - * Trade code size for speed on fixed-point multiplication. - * - * This speeds up repeated multiplication of the generator (that is, the - * multiplication in ECDSA signatures, and half of the multiplications in - * ECDSA verification and ECDHE) by a factor roughly 3 to 4. - * - * For each n-bit Short Weierstrass curve that is enabled, this adds 4n bytes - * of code size if n < 384 and 8n otherwise. - * - * Change this value to 0 to reduce code size. - */ -#define MBEDTLS_ECP_FIXED_POINT_OPTIM 1 /**< Enable fixed-point speed-up. */ -#endif /* MBEDTLS_ECP_FIXED_POINT_OPTIM */ - -/** \} name SECTION: Module settings */ - -#else /* MBEDTLS_ECP_ALT */ -#include "ecp_alt.h" -#endif /* MBEDTLS_ECP_ALT */ - -/** - * The maximum size of the groups, that is, of \c N and \c P. - */ -#if !defined(MBEDTLS_ECP_LIGHT) -/* Dummy definition to help code that has optional ECP support and - * defines an MBEDTLS_ECP_MAX_BYTES-sized array unconditionally. */ -#define MBEDTLS_ECP_MAX_BITS 1 -/* Note: the curves must be listed in DECREASING size! */ -#elif defined(MBEDTLS_ECP_DP_SECP521R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 521 -#elif defined(MBEDTLS_ECP_DP_BP512R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 512 -#elif defined(MBEDTLS_ECP_DP_CURVE448_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 448 -#elif defined(MBEDTLS_ECP_DP_BP384R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 384 -#elif defined(MBEDTLS_ECP_DP_SECP384R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 384 -#elif defined(MBEDTLS_ECP_DP_BP256R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 256 -#elif defined(MBEDTLS_ECP_DP_SECP256K1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 256 -#elif defined(MBEDTLS_ECP_DP_SECP256R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 256 -#elif defined(MBEDTLS_ECP_DP_CURVE25519_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 255 -#elif defined(MBEDTLS_ECP_DP_SECP224K1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 225 // n is slightly above 2^224 -#elif defined(MBEDTLS_ECP_DP_SECP224R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 224 -#elif defined(MBEDTLS_ECP_DP_SECP192K1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 192 -#elif defined(MBEDTLS_ECP_DP_SECP192R1_ENABLED) -#define MBEDTLS_ECP_MAX_BITS 192 -#else /* !MBEDTLS_ECP_LIGHT */ -#error "Missing definition of MBEDTLS_ECP_MAX_BITS" -#endif /* !MBEDTLS_ECP_LIGHT */ - -#define MBEDTLS_ECP_MAX_BYTES ((MBEDTLS_ECP_MAX_BITS + 7) / 8) -#define MBEDTLS_ECP_MAX_PT_LEN (2 * MBEDTLS_ECP_MAX_BYTES + 1) - -#if defined(MBEDTLS_ECP_RESTARTABLE) - -/** - * \brief Internal restart context for multiplication - * - * \note Opaque struct - */ -typedef struct mbedtls_ecp_restart_mul mbedtls_ecp_restart_mul_ctx; - -/** - * \brief Internal restart context for ecp_muladd() - * - * \note Opaque struct - */ -typedef struct mbedtls_ecp_restart_muladd mbedtls_ecp_restart_muladd_ctx; - -/** - * \brief General context for resuming ECC operations - */ -typedef struct { - unsigned MBEDTLS_PRIVATE(ops_done); /*!< current ops count */ - unsigned MBEDTLS_PRIVATE(depth); /*!< call depth (0 = top-level) */ - mbedtls_ecp_restart_mul_ctx *MBEDTLS_PRIVATE(rsm); /*!< ecp_mul_comb() sub-context */ - mbedtls_ecp_restart_muladd_ctx *MBEDTLS_PRIVATE(ma); /*!< ecp_muladd() sub-context */ -} mbedtls_ecp_restart_ctx; - -/* - * Operation counts for restartable functions - */ -#define MBEDTLS_ECP_OPS_CHK 3 /*!< basic ops count for ecp_check_pubkey() */ -#define MBEDTLS_ECP_OPS_DBL 8 /*!< basic ops count for ecp_double_jac() */ -#define MBEDTLS_ECP_OPS_ADD 11 /*!< basic ops count for see ecp_add_mixed() */ -#define MBEDTLS_ECP_OPS_INV 120 /*!< empirical equivalent for mpi_mod_inv() */ - -/** - * \brief Internal; for restartable functions in other modules. - * Check and update basic ops budget. - * - * \param grp Group structure - * \param rs_ctx Restart context - * \param ops Number of basic ops to do - * - * \return \c 0 if doing \p ops basic ops is still allowed, - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS otherwise. - */ -int mbedtls_ecp_check_budget(const mbedtls_ecp_group *grp, - mbedtls_ecp_restart_ctx *rs_ctx, - unsigned ops); - -/* Utility macro for checking and updating ops budget */ -#define MBEDTLS_ECP_BUDGET(ops) \ - MBEDTLS_MPI_CHK(mbedtls_ecp_check_budget(grp, rs_ctx, \ - (unsigned) (ops))); - -#else /* MBEDTLS_ECP_RESTARTABLE */ - -#define MBEDTLS_ECP_BUDGET(ops) /* no-op; for compatibility */ - -/* We want to declare restartable versions of existing functions anyway */ -typedef void mbedtls_ecp_restart_ctx; - -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -/** - * \brief The ECP key-pair structure. - * - * A generic key-pair that may be used for ECDSA and fixed ECDH, for example. - * - * \note Members are deliberately in the same order as in the - * ::mbedtls_ecdsa_context structure. - */ -typedef struct mbedtls_ecp_keypair { - mbedtls_ecp_group MBEDTLS_PRIVATE(grp); /*!< Elliptic curve and base point */ - mbedtls_mpi MBEDTLS_PRIVATE(d); /*!< our secret value */ - mbedtls_ecp_point MBEDTLS_PRIVATE(Q); /*!< our public value */ -} -mbedtls_ecp_keypair; - -/** - * The uncompressed point format for Short Weierstrass curves - * (MBEDTLS_ECP_DP_SECP_XXX and MBEDTLS_ECP_DP_BP_XXX). - */ -#define MBEDTLS_ECP_PF_UNCOMPRESSED 0 -/** - * The compressed point format for Short Weierstrass curves - * (MBEDTLS_ECP_DP_SECP_XXX and MBEDTLS_ECP_DP_BP_XXX). - * - * \warning While this format is supported for all concerned curves for - * writing, when it comes to parsing, it is not supported for all - * curves. Specifically, parsing compressed points on - * MBEDTLS_ECP_DP_SECP224R1 and MBEDTLS_ECP_DP_SECP224K1 is not - * supported. - */ -#define MBEDTLS_ECP_PF_COMPRESSED 1 - -/* - * Some other constants from RFC 4492 - */ -#define MBEDTLS_ECP_TLS_NAMED_CURVE 3 /**< The named_curve of ECCurveType. */ - -#if defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Set the maximum number of basic operations done in a row. - * - * If more operations are needed to complete a computation, - * #MBEDTLS_ERR_ECP_IN_PROGRESS will be returned by the - * function performing the computation. It is then the - * caller's responsibility to either call again with the same - * parameters until it returns 0 or an error code; or to free - * the restart context if the operation is to be aborted. - * - * It is strictly required that all input parameters and the - * restart context be the same on successive calls for the - * same operation, but output parameters need not be the - * same; they must not be used until the function finally - * returns 0. - * - * This only applies to functions whose documentation - * mentions they may return #MBEDTLS_ERR_ECP_IN_PROGRESS (or - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS for functions in the - * SSL module). For functions that accept a "restart context" - * argument, passing NULL disables restart and makes the - * function equivalent to the function with the same name - * with \c _restartable removed. For functions in the ECDH - * module, restart is disabled unless the function accepts - * an "ECDH context" argument and - * mbedtls_ecdh_enable_restart() was previously called on - * that context. For function in the SSL module, restart is - * only enabled for specific sides and key exchanges - * (currently only for clients and ECDHE-ECDSA). - * - * \warning Using the PSA interruptible interfaces with keys in local - * storage and no accelerator driver will also call this - * function to set the values specified via those interfaces, - * overwriting values previously set. Care should be taken if - * mixing these two interfaces. - * - * \param max_ops Maximum number of basic operations done in a row. - * Default: 0 (unlimited). - * Lower (non-zero) values mean ECC functions will block for - * a lesser maximum amount of time. - * - * \note A "basic operation" is defined as a rough equivalent of a - * multiplication in GF(p) for the NIST P-256 curve. - * As an indication, with default settings, a scalar - * multiplication (full run of \c mbedtls_ecp_mul()) is: - * - about 3300 basic operations for P-256 - * - about 9400 basic operations for P-384 - * - * \note Very low values are not always respected: sometimes - * functions need to block for a minimum number of - * operations, and will do so even if max_ops is set to a - * lower value. That minimum depends on the curve size, and - * can be made lower by decreasing the value of - * \c MBEDTLS_ECP_WINDOW_SIZE. As an indication, here is the - * lowest effective value for various curves and values of - * that parameter (w for short): - * w=6 w=5 w=4 w=3 w=2 - * P-256 208 208 160 136 124 - * P-384 682 416 320 272 248 - * P-521 1364 832 640 544 496 - * - * \note This setting is currently ignored by Curve25519. - */ -void mbedtls_ecp_set_max_ops(unsigned max_ops); - -/** - * \brief Check if restart is enabled (max_ops != 0) - * - * \return \c 0 if \c max_ops == 0 (restart disabled) - * \return \c 1 otherwise (restart enabled) - */ -int mbedtls_ecp_restart_is_enabled(void); -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -/* - * Get the type of a curve - */ -mbedtls_ecp_curve_type mbedtls_ecp_get_type(const mbedtls_ecp_group *grp); - -/** - * \brief This function retrieves the information defined in - * mbedtls_ecp_curve_info() for all supported curves. - * - * \note This function returns information about all curves - * supported by the library. Some curves may not be - * supported for all algorithms. Call mbedtls_ecdh_can_do() - * or mbedtls_ecdsa_can_do() to check if a curve is - * supported for ECDH or ECDSA. - * - * \return A statically allocated array. The last entry is 0. - */ -const mbedtls_ecp_curve_info *mbedtls_ecp_curve_list(void); - -/** - * \brief This function retrieves the list of internal group - * identifiers of all supported curves in the order of - * preference. - * - * \note This function returns information about all curves - * supported by the library. Some curves may not be - * supported for all algorithms. Call mbedtls_ecdh_can_do() - * or mbedtls_ecdsa_can_do() to check if a curve is - * supported for ECDH or ECDSA. - * - * \return A statically allocated array, - * terminated with MBEDTLS_ECP_DP_NONE. - */ -const mbedtls_ecp_group_id *mbedtls_ecp_grp_id_list(void); - -/** - * \brief This function retrieves curve information from an internal - * group identifier. - * - * \param grp_id An \c MBEDTLS_ECP_DP_XXX value. - * - * \return The associated curve information on success. - * \return NULL on failure. - */ -const mbedtls_ecp_curve_info *mbedtls_ecp_curve_info_from_grp_id(mbedtls_ecp_group_id grp_id); - -/** - * \brief This function retrieves curve information from a TLS - * NamedCurve value. - * - * \param tls_id An \c MBEDTLS_ECP_DP_XXX value. - * - * \return The associated curve information on success. - * \return NULL on failure. - */ -const mbedtls_ecp_curve_info *mbedtls_ecp_curve_info_from_tls_id(uint16_t tls_id); - -/** - * \brief This function retrieves curve information from a - * human-readable name. - * - * \param name The human-readable name. - * - * \return The associated curve information on success. - * \return NULL on failure. - */ -const mbedtls_ecp_curve_info *mbedtls_ecp_curve_info_from_name(const char *name); - -/** - * \brief This function initializes a point as zero. - * - * \param pt The point to initialize. - */ -void mbedtls_ecp_point_init(mbedtls_ecp_point *pt); - -/** - * \brief This function initializes an ECP group context - * without loading any domain parameters. - * - * \note After this function is called, domain parameters - * for various ECP groups can be loaded through the - * mbedtls_ecp_group_load() or mbedtls_ecp_tls_read_group() - * functions. - */ -void mbedtls_ecp_group_init(mbedtls_ecp_group *grp); - -/** - * \brief This function initializes a key pair as an invalid one. - * - * \param key The key pair to initialize. - */ -void mbedtls_ecp_keypair_init(mbedtls_ecp_keypair *key); - -/** - * \brief This function frees the components of a point. - * - * \param pt The point to free. - */ -void mbedtls_ecp_point_free(mbedtls_ecp_point *pt); - -/** - * \brief This function frees the components of an ECP group. - * - * \param grp The group to free. This may be \c NULL, in which - * case this function returns immediately. If it is not - * \c NULL, it must point to an initialized ECP group. - */ -void mbedtls_ecp_group_free(mbedtls_ecp_group *grp); - -/** - * \brief This function frees the components of a key pair. - * - * \param key The key pair to free. This may be \c NULL, in which - * case this function returns immediately. If it is not - * \c NULL, it must point to an initialized ECP key pair. - */ -void mbedtls_ecp_keypair_free(mbedtls_ecp_keypair *key); - -#if defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Initialize a restart context. - * - * \param ctx The restart context to initialize. This must - * not be \c NULL. - */ -void mbedtls_ecp_restart_init(mbedtls_ecp_restart_ctx *ctx); - -/** - * \brief Free the components of a restart context. - * - * \param ctx The restart context to free. This may be \c NULL, in which - * case this function returns immediately. If it is not - * \c NULL, it must point to an initialized restart context. - */ -void mbedtls_ecp_restart_free(mbedtls_ecp_restart_ctx *ctx); -#endif /* MBEDTLS_ECP_RESTARTABLE */ - -/** - * \brief This function copies the contents of point \p Q into - * point \p P. - * - * \param P The destination point. This must be initialized. - * \param Q The source point. This must be initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return Another negative error code for other kinds of failure. - */ -int mbedtls_ecp_copy(mbedtls_ecp_point *P, const mbedtls_ecp_point *Q); - -/** - * \brief This function copies the contents of group \p src into - * group \p dst. - * - * \param dst The destination group. This must be initialized. - * \param src The source group. This must be initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_group_copy(mbedtls_ecp_group *dst, - const mbedtls_ecp_group *src); - -/** - * \brief This function sets a point to the point at infinity. - * - * \param pt The point to set. This must be initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_set_zero(mbedtls_ecp_point *pt); - -/** - * \brief This function checks if a point is the point at infinity. - * - * \param pt The point to test. This must be initialized. - * - * \return \c 1 if the point is zero. - * \return \c 0 if the point is non-zero. - * \return A negative error code on failure. - */ -int mbedtls_ecp_is_zero(mbedtls_ecp_point *pt); - -/** - * \brief This function compares two points. - * - * \note This assumes that the points are normalized. Otherwise, - * they may compare as "not equal" even if they are. - * - * \param P The first point to compare. This must be initialized. - * \param Q The second point to compare. This must be initialized. - * - * \return \c 0 if the points are equal. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if the points are not equal. - */ -int mbedtls_ecp_point_cmp(const mbedtls_ecp_point *P, - const mbedtls_ecp_point *Q); - -/** - * \brief This function imports a non-zero point from two ASCII - * strings. - * - * \param P The destination point. This must be initialized. - * \param radix The numeric base of the input. - * \param x The first affine coordinate, as a null-terminated string. - * \param y The second affine coordinate, as a null-terminated string. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_MPI_XXX error code on failure. - */ -int mbedtls_ecp_point_read_string(mbedtls_ecp_point *P, int radix, - const char *x, const char *y); - -/** - * \brief This function exports a point into unsigned binary data. - * - * \param grp The group to which the point should belong. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param P The point to export. This must be initialized. - * \param format The point format. This must be either - * #MBEDTLS_ECP_PF_COMPRESSED or #MBEDTLS_ECP_PF_UNCOMPRESSED. - * (For groups without these formats, this parameter is - * ignored. But it still has to be either of the above - * values.) - * \param olen The address at which to store the length of - * the output in Bytes. This must not be \c NULL. - * \param buf The output buffer. This must be a writable buffer - * of length \p buflen Bytes. - * \param buflen The length of the output buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL if the output buffer - * is too small to hold the point. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the point format - * or the export for the given group is not implemented. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_point_write_binary(const mbedtls_ecp_group *grp, - const mbedtls_ecp_point *P, - int format, size_t *olen, - unsigned char *buf, size_t buflen); - -/** - * \brief This function imports a point from unsigned binary data. - * - * \note This function does not check that the point actually - * belongs to the given group, see mbedtls_ecp_check_pubkey() - * for that. - * - * \note For compressed points, see #MBEDTLS_ECP_PF_COMPRESSED for - * limitations. - * - * \param grp The group to which the point should belong. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param P The destination context to import the point to. - * This must be initialized. - * \param buf The input buffer. This must be a readable buffer - * of length \p ilen Bytes. - * \param ilen The length of the input buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if the input is invalid. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the import for the - * given group is not implemented. - */ -int mbedtls_ecp_point_read_binary(const mbedtls_ecp_group *grp, - mbedtls_ecp_point *P, - const unsigned char *buf, size_t ilen); - -/** - * \brief This function imports a point from a TLS ECPoint record. - * - * \note On function return, \p *buf is updated to point immediately - * after the ECPoint record. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param pt The destination point. - * \param buf The address of the pointer to the start of the input buffer. - * \param len The length of the buffer. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_MPI_XXX error code on initialization - * failure. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if input is invalid. - */ -int mbedtls_ecp_tls_read_point(const mbedtls_ecp_group *grp, - mbedtls_ecp_point *pt, - const unsigned char **buf, size_t len); - -/** - * \brief This function exports a point as a TLS ECPoint record - * defined in RFC 4492, Section 5.4. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param pt The point to be exported. This must be initialized. - * \param format The point format to use. This must be either - * #MBEDTLS_ECP_PF_COMPRESSED or #MBEDTLS_ECP_PF_UNCOMPRESSED. - * \param olen The address at which to store the length in Bytes - * of the data written. - * \param buf The target buffer. This must be a writable buffer of - * length \p blen Bytes. - * \param blen The length of the target buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if the input is invalid. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL if the target buffer - * is too small to hold the exported point. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_tls_write_point(const mbedtls_ecp_group *grp, - const mbedtls_ecp_point *pt, - int format, size_t *olen, - unsigned char *buf, size_t blen); - -/** - * \brief This function sets up an ECP group context - * from a standardized set of domain parameters. - * - * \note The index should be a value of the NamedCurve enum, - * as defined in RFC-4492: Elliptic Curve Cryptography - * (ECC) Cipher Suites for Transport Layer Security (TLS), - * usually in the form of an \c MBEDTLS_ECP_DP_XXX macro. - * - * \param grp The group context to setup. This must be initialized. - * \param id The identifier of the domain parameter set to load. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if \p id doesn't - * correspond to a known group. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_group_load(mbedtls_ecp_group *grp, mbedtls_ecp_group_id id); - -/** - * \brief This function sets up an ECP group context from a TLS - * ECParameters record as defined in RFC 4492, Section 5.4. - * - * \note The read pointer \p buf is updated to point right after - * the ECParameters record on exit. - * - * \param grp The group context to setup. This must be initialized. - * \param buf The address of the pointer to the start of the input buffer. - * \param len The length of the input buffer \c *buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if input is invalid. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the group is not - * recognized. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_tls_read_group(mbedtls_ecp_group *grp, - const unsigned char **buf, size_t len); - -/** - * \brief This function extracts an elliptic curve group ID from a - * TLS ECParameters record as defined in RFC 4492, Section 5.4. - * - * \note The read pointer \p buf is updated to point right after - * the ECParameters record on exit. - * - * \param grp The address at which to store the group id. - * This must not be \c NULL. - * \param buf The address of the pointer to the start of the input buffer. - * \param len The length of the input buffer \c *buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if input is invalid. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the group is not - * recognized. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_tls_read_group_id(mbedtls_ecp_group_id *grp, - const unsigned char **buf, - size_t len); -/** - * \brief This function exports an elliptic curve as a TLS - * ECParameters record as defined in RFC 4492, Section 5.4. - * - * \param grp The ECP group to be exported. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param olen The address at which to store the number of Bytes written. - * This must not be \c NULL. - * \param buf The buffer to write to. This must be a writable buffer - * of length \p blen Bytes. - * \param blen The length of the output buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL if the output - * buffer is too small to hold the exported group. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_tls_write_group(const mbedtls_ecp_group *grp, - size_t *olen, - unsigned char *buf, size_t blen); - -/** - * \brief This function performs a scalar multiplication of a point - * by an integer: \p R = \p m * \p P. - * - * It is not thread-safe to use same group in multiple threads. - * - * \note To prevent timing attacks, this function - * executes the exact same sequence of base-field - * operations for any valid \p m. It avoids any if-branch or - * array index depending on the value of \p m. It also uses - * \p f_rng to randomize some intermediate results. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param R The point in which to store the result of the calculation. - * This must be initialized. - * \param m The integer by which to multiply. This must be initialized. - * \param P The point to multiply. This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c - * NULL if \p f_rng doesn't need a context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if \p m is not a valid private - * key, or \p P is not a valid public key. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_mul(mbedtls_ecp_group *grp, mbedtls_ecp_point *R, - const mbedtls_mpi *m, const mbedtls_ecp_point *P, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** - * \brief This function performs multiplication of a point by - * an integer: \p R = \p m * \p P in a restartable way. - * - * \see mbedtls_ecp_mul() - * - * \note This function does the same as \c mbedtls_ecp_mul(), but - * it can return early and restart according to the limit set - * with \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param R The point in which to store the result of the calculation. - * This must be initialized. - * \param m The integer by which to multiply. This must be initialized. - * \param P The point to multiply. This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c - * NULL if \p f_rng doesn't need a context. - * \param rs_ctx The restart context (NULL disables restart). - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if \p m is not a valid private - * key, or \p P is not a valid public key. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_mul_restartable(mbedtls_ecp_group *grp, mbedtls_ecp_point *R, - const mbedtls_mpi *m, const mbedtls_ecp_point *P, - mbedtls_f_rng_t *f_rng, void *p_rng, - mbedtls_ecp_restart_ctx *rs_ctx); - -#if defined(MBEDTLS_ECP_SHORT_WEIERSTRASS_ENABLED) -/** - * \brief This function checks if domain parameter A of the curve is - * \c -3. - * - * \note This function is only defined for short Weierstrass curves. - * It may not be included in builds without any short - * Weierstrass curve. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * - * \return \c 1 if A = -3. - * \return \c 0 Otherwise. - */ -static inline int mbedtls_ecp_group_a_is_minus_3(const mbedtls_ecp_group *grp) -{ - return grp->A.MBEDTLS_PRIVATE(p) == NULL; -} - -/** - * \brief This function performs multiplication and addition of two - * points by integers: \p R = \p m * \p P + \p n * \p Q - * - * It is not thread-safe to use same group in multiple threads. - * - * \note In contrast to mbedtls_ecp_mul(), this function does not - * guarantee a constant execution flow and timing. - * - * \note This function is only defined for short Weierstrass curves. - * It may not be included in builds without any short - * Weierstrass curve. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param R The point in which to store the result of the calculation. - * This must be initialized. - * \param m The integer by which to multiply \p P. - * This must be initialized. - * \param P The point to multiply by \p m. This must be initialized. - * \param n The integer by which to multiply \p Q. - * This must be initialized. - * \param Q The point to be multiplied by \p n. - * This must be initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if \p m or \p n are not - * valid private keys, or \p P or \p Q are not valid public - * keys. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if \p grp does not - * designate a short Weierstrass curve. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_muladd(mbedtls_ecp_group *grp, mbedtls_ecp_point *R, - const mbedtls_mpi *m, const mbedtls_ecp_point *P, - const mbedtls_mpi *n, const mbedtls_ecp_point *Q); - -/** - * \brief This function performs multiplication and addition of two - * points by integers: \p R = \p m * \p P + \p n * \p Q in a - * restartable way. - * - * \see \c mbedtls_ecp_muladd() - * - * \note This function works the same as \c mbedtls_ecp_muladd(), - * but it can return early and restart according to the limit - * set with \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \note This function is only defined for short Weierstrass curves. - * It may not be included in builds without any short - * Weierstrass curve. - * - * \param grp The ECP group to use. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param R The point in which to store the result of the calculation. - * This must be initialized. - * \param m The integer by which to multiply \p P. - * This must be initialized. - * \param P The point to multiply by \p m. This must be initialized. - * \param n The integer by which to multiply \p Q. - * This must be initialized. - * \param Q The point to be multiplied by \p n. - * This must be initialized. - * \param rs_ctx The restart context (NULL disables restart). - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if \p m or \p n are not - * valid private keys, or \p P or \p Q are not valid public - * keys. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if \p grp does not - * designate a short Weierstrass curve. - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_muladd_restartable( - mbedtls_ecp_group *grp, mbedtls_ecp_point *R, - const mbedtls_mpi *m, const mbedtls_ecp_point *P, - const mbedtls_mpi *n, const mbedtls_ecp_point *Q, - mbedtls_ecp_restart_ctx *rs_ctx); -#endif /* MBEDTLS_ECP_SHORT_WEIERSTRASS_ENABLED */ - -/** - * \brief This function checks that a point is a valid public key - * on this curve. - * - * It only checks that the point is non-zero, has - * valid coordinates and lies on the curve. It does not verify - * that it is indeed a multiple of \c G. This additional - * check is computationally more expensive, is not required - * by standards, and should not be necessary if the group - * used has a small cofactor. In particular, it is useless for - * the NIST groups which all have a cofactor of 1. - * - * \note This function uses bare components rather than an - * ::mbedtls_ecp_keypair structure, to ease use with other - * structures, such as ::mbedtls_ecdh_context or - * ::mbedtls_ecdsa_context. - * - * \param grp The ECP group the point should belong to. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param pt The point to check. This must be initialized. - * - * \return \c 0 if the point is a valid public key. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if the point is not - * a valid public key for the given curve. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_check_pubkey(const mbedtls_ecp_group *grp, - const mbedtls_ecp_point *pt); - -/** - * \brief This function checks that an \c mbedtls_mpi is a - * valid private key for this curve. - * - * \note This function uses bare components rather than an - * ::mbedtls_ecp_keypair structure to ease use with other - * structures, such as ::mbedtls_ecdh_context or - * ::mbedtls_ecdsa_context. - * - * \param grp The ECP group the private key should belong to. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param d The integer to check. This must be initialized. - * - * \return \c 0 if the point is a valid private key. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY if the point is not a valid - * private key for the given curve. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_check_privkey(const mbedtls_ecp_group *grp, - const mbedtls_mpi *d); - -/** - * \brief This function generates a private key. - * - * \param grp The ECP group to generate a private key for. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param d The destination MPI (secret part). This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG parameter to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX error code - * on failure. - */ -int mbedtls_ecp_gen_privkey(const mbedtls_ecp_group *grp, - mbedtls_mpi *d, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function generates a keypair with a configurable base - * point. - * - * \note This function uses bare components rather than an - * ::mbedtls_ecp_keypair structure to ease use with other - * structures, such as ::mbedtls_ecdh_context or - * ::mbedtls_ecdsa_context. - * - * \param grp The ECP group to generate a key pair for. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param G The base point to use. This must be initialized - * and belong to \p grp. It replaces the default base - * point \c grp->G used by mbedtls_ecp_gen_keypair(). - * \param d The destination MPI (secret part). - * This must be initialized. - * \param Q The destination point (public part). - * This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may - * be \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX error code - * on failure. - */ -int mbedtls_ecp_gen_keypair_base(mbedtls_ecp_group *grp, - const mbedtls_ecp_point *G, - mbedtls_mpi *d, mbedtls_ecp_point *Q, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function generates an ECP keypair. - * - * \note This function uses bare components rather than an - * ::mbedtls_ecp_keypair structure to ease use with other - * structures, such as ::mbedtls_ecdh_context or - * ::mbedtls_ecdsa_context. - * - * \param grp The ECP group to generate a key pair for. - * This must be initialized and have group parameters - * set, for example through mbedtls_ecp_group_load(). - * \param d The destination MPI (secret part). - * This must be initialized. - * \param Q The destination point (public part). - * This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may - * be \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX error code - * on failure. - */ -int mbedtls_ecp_gen_keypair(mbedtls_ecp_group *grp, mbedtls_mpi *d, - mbedtls_ecp_point *Q, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief This function generates an ECP key. - * - * \param grp_id The ECP group identifier. - * \param key The destination key. This must be initialized. - * \param f_rng The RNG function to use. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may - * be \c NULL if \p f_rng doesn't need a context argument. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_ECP_XXX or \c MBEDTLS_MPI_XXX error code - * on failure. - */ -int mbedtls_ecp_gen_key(mbedtls_ecp_group_id grp_id, mbedtls_ecp_keypair *key, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** \brief Set the public key in a key pair object. - * - * \note This function does not check that the point actually - * belongs to the given group. Call mbedtls_ecp_check_pubkey() - * on \p Q before calling this function to check that. - * - * \note This function does not check that the public key matches - * the private key that is already in \p key, if any. - * To check the consistency of the resulting key pair object, - * call mbedtls_ecp_check_pub_priv() after setting both - * the public key and the private key. - * - * \param grp_id The ECP group identifier. - * \param key The key pair object. It must be initialized. - * If its group has already been set, it must match \p grp_id. - * If its group has not been set, it will be set to \p grp_id. - * If the public key has already been set, it is overwritten. - * \param Q The public key to copy. This must be a point on the - * curve indicated by \p grp_id. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if \p key does not - * match \p grp_id. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the operation for - * the group is not implemented. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_set_public_key(mbedtls_ecp_group_id grp_id, - mbedtls_ecp_keypair *key, - const mbedtls_ecp_point *Q); - -/** - * \brief This function reads an elliptic curve private key. - * - * \note This function does not set the public key in the - * key pair object. Without a public key, the key pair object - * cannot be used with operations that require the public key. - * Call mbedtls_ecp_keypair_calc_public() to set the public - * key from the private key. Alternatively, you can call - * mbedtls_ecp_set_public_key() to set the public key part, - * and then optionally mbedtls_ecp_check_pub_priv() to check - * that the private and public parts are consistent. - * - * \note If a public key has already been set in the key pair - * object, this function does not check that it is consistent - * with the private key. Call mbedtls_ecp_check_pub_priv() - * after setting both the public key and the private key - * to make that check. - * - * \param grp_id The ECP group identifier. - * \param key The destination key. - * \param buf The buffer containing the binary representation of the - * key. (Big endian integer for Weierstrass curves, byte - * string for Montgomery curves.) - * \param buflen The length of the buffer in bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_INVALID_KEY error if the key is - * invalid. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED if memory allocation failed. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the operation for - * the group is not implemented. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_ecp_read_key(mbedtls_ecp_group_id grp_id, mbedtls_ecp_keypair *key, - const unsigned char *buf, size_t buflen); - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief This function exports an elliptic curve private key. - * - * \deprecated Note that although this function accepts an output - * buffer that is smaller or larger than the key, most key - * import interfaces require the output to have exactly - * key's nominal length. It is generally simplest to - * pass the key's nominal length as \c buflen, after - * checking that the output buffer is large enough. - * See the description of the \p buflen parameter for - * how to calculate the nominal length. - * To avoid this difficulty, use mbedtls_ecp_write_key_ext() - * instead. - * mbedtls_ecp_write_key() is deprecated and will be - * removed in a future version of the library. - * - * \note If the private key was not set in \p key, - * the output is unspecified. Future versions - * may return an error in that case. - * - * \param key The private key. - * \param buf The output buffer for containing the binary representation - * of the key. - * For Weierstrass curves, this is the big-endian - * representation, padded with null bytes at the beginning - * to reach \p buflen bytes. - * For Montgomery curves, this is the standard byte string - * representation (which is little-endian), padded with - * null bytes at the end to reach \p buflen bytes. - * \param buflen The total length of the buffer in bytes. - * The length of the output is - * (`grp->nbits` + 7) / 8 bytes - * where `grp->nbits` is the private key size in bits. - * For Weierstrass keys, if the output buffer is smaller, - * leading zeros are trimmed to fit if possible. For - * Montgomery keys, the output buffer must always be large - * enough for the nominal length. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL or - * #MBEDTLS_ERR_MPI_BUFFER_TOO_SMALL if the \p key - * representation is larger than the available space in \p buf. - * \return Another negative error code on different kinds of failure. - */ -int MBEDTLS_DEPRECATED mbedtls_ecp_write_key(mbedtls_ecp_keypair *key, - unsigned char *buf, size_t buflen); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/** - * \brief This function exports an elliptic curve private key. - * - * \param key The private key. - * \param olen On success, the length of the private key. - * This is always (`grp->nbits` + 7) / 8 bytes - * where `grp->nbits` is the private key size in bits. - * \param buf The output buffer for containing the binary representation - * of the key. - * \param buflen The total length of the buffer in bytes. - * #MBEDTLS_ECP_MAX_BYTES is always sufficient. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL if the \p key - * representation is larger than the available space in \p buf. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if no private key is - * set in \p key. - * \return Another negative error code on different kinds of failure. - */ -int mbedtls_ecp_write_key_ext(const mbedtls_ecp_keypair *key, - size_t *olen, unsigned char *buf, size_t buflen); - -/** - * \brief This function exports an elliptic curve public key. - * - * \note If the public key was not set in \p key, - * the output is unspecified. Future versions - * may return an error in that case. - * - * \param key The public key. - * \param format The point format. This must be either - * #MBEDTLS_ECP_PF_COMPRESSED or #MBEDTLS_ECP_PF_UNCOMPRESSED. - * (For groups without these formats, this parameter is - * ignored. But it still has to be either of the above - * values.) - * \param olen The address at which to store the length of - * the output in Bytes. This must not be \c NULL. - * \param buf The output buffer. This must be a writable buffer - * of length \p buflen Bytes. - * \param buflen The length of the output buffer \p buf in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_ECP_BUFFER_TOO_SMALL if the output buffer - * is too small to hold the point. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if the point format - * or the export for the given group is not implemented. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_write_public_key(const mbedtls_ecp_keypair *key, - int format, size_t *olen, - unsigned char *buf, size_t buflen); - -/** - * \brief This function checks that the keypair objects - * \p pub and \p prv have the same group and the - * same public point, and that the private key in - * \p prv is consistent with the public key. - * - * \param pub The keypair structure holding the public key. This - * must be initialized. If it contains a private key, that - * part is ignored. - * \param prv The keypair structure holding the full keypair. - * This must be initialized. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c - * NULL if \p f_rng doesn't need a context. - * - * \return \c 0 on success, meaning that the keys are valid and match. - * \return #MBEDTLS_ERR_ECP_BAD_INPUT_DATA if the keys are invalid or do not match. - * \return An \c MBEDTLS_ERR_ECP_XXX or an \c MBEDTLS_ERR_MPI_XXX - * error code on calculation failure. - */ -int mbedtls_ecp_check_pub_priv( - const mbedtls_ecp_keypair *pub, const mbedtls_ecp_keypair *prv, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** \brief Calculate the public key from a private key in a key pair. - * - * \param key A keypair structure. It must have a private key set. - * If the public key is set, it will be overwritten. - * \param f_rng The RNG function. This must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c - * NULL if \p f_rng doesn't need a context. - * - * \return \c 0 on success. The key pair object can be used for - * operations that require the public key. - * \return An \c MBEDTLS_ERR_ECP_XXX or an \c MBEDTLS_ERR_MPI_XXX - * error code on calculation failure. - */ -int mbedtls_ecp_keypair_calc_public( - mbedtls_ecp_keypair *key, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** \brief Query the group that a key pair belongs to. - * - * \param key The key pair to query. - * - * \return The group ID for the group registered in the key pair - * object. - * This is \c MBEDTLS_ECP_DP_NONE if no group has been set - * in the key pair object. - */ -mbedtls_ecp_group_id mbedtls_ecp_keypair_get_group_id( - const mbedtls_ecp_keypair *key); - -/** - * \brief This function exports generic key-pair parameters. - * - * Each of the output parameters can be a null pointer - * if you do not need that parameter. - * - * \note If the private key or the public key was not set in \p key, - * the corresponding output is unspecified. Future versions - * may return an error in that case. - * - * \param key The key pair to export from. - * \param grp Slot for exported ECP group. - * It must either be null or point to an initialized ECP group. - * \param d Slot for the exported secret value. - * It must either be null or point to an initialized mpi. - * \param Q Slot for the exported public value. - * It must either be null or point to an initialized ECP point. - * - * \return \c 0 on success, - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory-allocation failure. - * \return #MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE if key id doesn't - * correspond to a known group. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ecp_export(const mbedtls_ecp_keypair *key, mbedtls_ecp_group *grp, - mbedtls_mpi *d, mbedtls_ecp_point *Q); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The ECP checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_ecp_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* ecp.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/entropy.h b/android/mbedtls/arm64-v8a/include/mbedtls/entropy.h deleted file mode 100644 index 6c64e3e4e..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/entropy.h +++ /dev/null @@ -1,274 +0,0 @@ -/** - * \file entropy.h - * - * \brief Entropy accumulator implementation - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_ENTROPY_H -#define MBEDTLS_ENTROPY_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include - -#include "md.h" - -#if (defined(MBEDTLS_MD_CAN_SHA512) || defined(PSA_WANT_ALG_SHA_512)) && \ - !defined(MBEDTLS_ENTROPY_FORCE_SHA256) -#define MBEDTLS_ENTROPY_SHA512_ACCUMULATOR -#define MBEDTLS_ENTROPY_MD MBEDTLS_MD_SHA512 -#define MBEDTLS_ENTROPY_BLOCK_SIZE 64 /**< Block size of entropy accumulator (SHA-512) */ -#else -#if (defined(MBEDTLS_MD_CAN_SHA256) || defined(PSA_WANT_ALG_SHA_256)) -#define MBEDTLS_ENTROPY_SHA256_ACCUMULATOR -#define MBEDTLS_ENTROPY_MD MBEDTLS_MD_SHA256 -#define MBEDTLS_ENTROPY_BLOCK_SIZE 32 /**< Block size of entropy accumulator (SHA-256) */ -#endif -#endif - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - - -/** Critical entropy source failure. */ -#define MBEDTLS_ERR_ENTROPY_SOURCE_FAILED -0x003C -/** No more sources can be added. */ -#define MBEDTLS_ERR_ENTROPY_MAX_SOURCES -0x003E -/** No sources have been added to poll. */ -#define MBEDTLS_ERR_ENTROPY_NO_SOURCES_DEFINED -0x0040 -/** No strong sources have been added to poll. */ -#define MBEDTLS_ERR_ENTROPY_NO_STRONG_SOURCE -0x003D -/** Read/write error in file. */ -#define MBEDTLS_ERR_ENTROPY_FILE_IO_ERROR -0x003F - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_ENTROPY_MAX_SOURCES) -#define MBEDTLS_ENTROPY_MAX_SOURCES 20 /**< Maximum number of sources supported */ -#endif - -#if !defined(MBEDTLS_ENTROPY_MAX_GATHER) -#define MBEDTLS_ENTROPY_MAX_GATHER 128 /**< Maximum amount requested from entropy sources */ -#endif - -/** \} name SECTION: Module settings */ - -#define MBEDTLS_ENTROPY_MAX_SEED_SIZE 1024 /**< Maximum size of seed we read from seed file */ -#define MBEDTLS_ENTROPY_SOURCE_MANUAL MBEDTLS_ENTROPY_MAX_SOURCES - -#define MBEDTLS_ENTROPY_SOURCE_STRONG 1 /**< Entropy source is strong */ -#define MBEDTLS_ENTROPY_SOURCE_WEAK 0 /**< Entropy source is weak */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Entropy poll callback pointer - * - * \param data Callback-specific data pointer - * \param output Data to fill - * \param len Maximum size to provide - * \param olen The actual amount of bytes put into the buffer (Can be 0) - * - * \return 0 if no critical failures occurred, - * MBEDTLS_ERR_ENTROPY_SOURCE_FAILED otherwise - */ -typedef int (*mbedtls_entropy_f_source_ptr)(void *data, unsigned char *output, size_t len, - size_t *olen); - -/** - * \brief Entropy source state - */ -typedef struct mbedtls_entropy_source_state { - mbedtls_entropy_f_source_ptr MBEDTLS_PRIVATE(f_source); /**< The entropy source callback */ - void *MBEDTLS_PRIVATE(p_source); /**< The callback data pointer */ - size_t MBEDTLS_PRIVATE(size); /**< Amount received in bytes */ - size_t MBEDTLS_PRIVATE(threshold); /**< Minimum bytes required before release */ - int MBEDTLS_PRIVATE(strong); /**< Is the source strong? */ -} -mbedtls_entropy_source_state; - -/** - * \brief Entropy context structure - */ -typedef struct mbedtls_entropy_context { - mbedtls_md_context_t MBEDTLS_PRIVATE(accumulator); - int MBEDTLS_PRIVATE(accumulator_started); /* 0 after init. - * 1 after the first update. - * -1 after free. */ - int MBEDTLS_PRIVATE(source_count); /* Number of entries used in source. */ - mbedtls_entropy_source_state MBEDTLS_PRIVATE(source)[MBEDTLS_ENTROPY_MAX_SOURCES]; -#if defined(MBEDTLS_THREADING_C) - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); /*!< mutex */ -#endif -#if defined(MBEDTLS_ENTROPY_NV_SEED) - int MBEDTLS_PRIVATE(initial_entropy_run); -#endif -} -mbedtls_entropy_context; - -#if !defined(MBEDTLS_NO_PLATFORM_ENTROPY) -/** - * \brief Platform-specific entropy poll callback - */ -int mbedtls_platform_entropy_poll(void *data, - unsigned char *output, size_t len, size_t *olen); -#endif - -/** - * \brief Initialize the context - * - * \param ctx Entropy context to initialize - */ -void mbedtls_entropy_init(mbedtls_entropy_context *ctx); - -/** - * \brief Free the data in the context - * - * \param ctx Entropy context to free - */ -void mbedtls_entropy_free(mbedtls_entropy_context *ctx); - -/** - * \brief Adds an entropy source to poll - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param ctx Entropy context - * \param f_source Entropy function - * \param p_source Function data - * \param threshold Minimum required from source before entropy is released - * ( with mbedtls_entropy_func() ) (in bytes) - * \param strong MBEDTLS_ENTROPY_SOURCE_STRONG or - * MBEDTLS_ENTROPY_SOURCE_WEAK. - * At least one strong source needs to be added. - * Weaker sources (such as the cycle counter) can be used as - * a complement. - * - * \return 0 if successful or MBEDTLS_ERR_ENTROPY_MAX_SOURCES - */ -int mbedtls_entropy_add_source(mbedtls_entropy_context *ctx, - mbedtls_entropy_f_source_ptr f_source, void *p_source, - size_t threshold, int strong); - -/** - * \brief Trigger an extra gather poll for the accumulator - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param ctx Entropy context - * - * \return 0 if successful, or MBEDTLS_ERR_ENTROPY_SOURCE_FAILED - */ -int mbedtls_entropy_gather(mbedtls_entropy_context *ctx); - -/** - * \brief Retrieve entropy from the accumulator - * (Maximum length: MBEDTLS_ENTROPY_BLOCK_SIZE) - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param data Entropy context - * \param output Buffer to fill - * \param len Number of bytes desired, must be at most MBEDTLS_ENTROPY_BLOCK_SIZE - * - * \return 0 if successful, or MBEDTLS_ERR_ENTROPY_SOURCE_FAILED - */ -int mbedtls_entropy_func(void *data, unsigned char *output, size_t len); - -/** - * \brief Add data to the accumulator manually - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param ctx Entropy context - * \param data Data to add - * \param len Length of data - * - * \return 0 if successful - */ -int mbedtls_entropy_update_manual(mbedtls_entropy_context *ctx, - const unsigned char *data, size_t len); - -#if defined(MBEDTLS_ENTROPY_NV_SEED) -/** - * \brief Trigger an update of the seed file in NV by using the - * current entropy pool. - * - * \param ctx Entropy context - * - * \return 0 if successful - */ -int mbedtls_entropy_update_nv_seed(mbedtls_entropy_context *ctx); -#endif /* MBEDTLS_ENTROPY_NV_SEED */ - -#if defined(MBEDTLS_FS_IO) -/** - * \brief Write a seed file - * - * \param ctx Entropy context - * \param path Name of the file - * - * \return 0 if successful, - * MBEDTLS_ERR_ENTROPY_FILE_IO_ERROR on file error, or - * MBEDTLS_ERR_ENTROPY_SOURCE_FAILED - */ -int mbedtls_entropy_write_seed_file(mbedtls_entropy_context *ctx, const char *path); - -/** - * \brief Read and update a seed file. Seed is added to this - * instance. No more than MBEDTLS_ENTROPY_MAX_SEED_SIZE bytes are - * read from the seed file. The rest is ignored. - * - * \param ctx Entropy context - * \param path Name of the file - * - * \return 0 if successful, - * MBEDTLS_ERR_ENTROPY_FILE_IO_ERROR on file error, - * MBEDTLS_ERR_ENTROPY_SOURCE_FAILED - */ -int mbedtls_entropy_update_seed_file(mbedtls_entropy_context *ctx, const char *path); -#endif /* MBEDTLS_FS_IO */ - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine - * - * This module self-test also calls the entropy self-test, - * mbedtls_entropy_source_self_test(); - * - * \return 0 if successful, or 1 if a test failed - */ -int mbedtls_entropy_self_test(int verbose); - -#if defined(MBEDTLS_ENTROPY_HARDWARE_ALT) -/** - * \brief Checkup routine - * - * Verifies the integrity of the hardware entropy source - * provided by the function 'mbedtls_hardware_poll()'. - * - * Note this is the only hardware entropy source that is known - * at link time, and other entropy sources configured - * dynamically at runtime by the function - * mbedtls_entropy_add_source() will not be tested. - * - * \return 0 if successful, or 1 if a test failed - */ -int mbedtls_entropy_source_self_test(int verbose); -#endif /* MBEDTLS_ENTROPY_HARDWARE_ALT */ -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* entropy.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/error.h b/android/mbedtls/arm64-v8a/include/mbedtls/error.h deleted file mode 100644 index 635f7cd84..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/error.h +++ /dev/null @@ -1,201 +0,0 @@ -/** - * \file error.h - * - * \brief Error to string translation - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_ERROR_H -#define MBEDTLS_ERROR_H - -#include "mbedtls/build_info.h" - -#include - -/** - * Error code layout. - * - * Currently we try to keep all error codes within the negative space of 16 - * bits signed integers to support all platforms (-0x0001 - -0x7FFF). In - * addition we'd like to give two layers of information on the error if - * possible. - * - * For that purpose the error codes are segmented in the following manner: - * - * 16 bit error code bit-segmentation - * - * 1 bit - Unused (sign bit) - * 3 bits - High level module ID - * 5 bits - Module-dependent error code - * 7 bits - Low level module errors - * - * For historical reasons, low-level error codes are divided in even and odd, - * even codes were assigned first, and -1 is reserved for other errors. - * - * Low-level module errors (0x0002-0x007E, 0x0001-0x007F) - * - * Module Nr Codes assigned - * ERROR 2 0x006E 0x0001 - * MPI 7 0x0002-0x0010 - * GCM 3 0x0012-0x0016 0x0013-0x0013 - * THREADING 3 0x001A-0x001E - * AES 5 0x0020-0x0022 0x0021-0x0025 - * CAMELLIA 3 0x0024-0x0026 0x0027-0x0027 - * BASE64 2 0x002A-0x002C - * OID 1 0x002E-0x002E 0x000B-0x000B - * PADLOCK 1 0x0030-0x0030 - * DES 2 0x0032-0x0032 0x0033-0x0033 - * CTR_DBRG 4 0x0034-0x003A - * ENTROPY 3 0x003C-0x0040 0x003D-0x003F - * NET 13 0x0042-0x0052 0x0043-0x0049 - * ARIA 4 0x0058-0x005E - * ASN1 7 0x0060-0x006C - * CMAC 1 0x007A-0x007A - * PBKDF2 1 0x007C-0x007C - * HMAC_DRBG 4 0x0003-0x0009 - * CCM 3 0x000D-0x0011 - * MD5 1 0x002F-0x002F - * RIPEMD160 1 0x0031-0x0031 - * SHA1 1 0x0035-0x0035 0x0073-0x0073 - * SHA256 1 0x0037-0x0037 0x0074-0x0074 - * SHA512 1 0x0039-0x0039 0x0075-0x0075 - * SHA-3 1 0x0076-0x0076 - * CHACHA20 3 0x0051-0x0055 - * POLY1305 3 0x0057-0x005B - * CHACHAPOLY 2 0x0054-0x0056 - * PLATFORM 2 0x0070-0x0072 - * LMS 5 0x0011-0x0019 - * - * High-level module nr (3 bits - 0x0...-0x7...) - * Name ID Nr of Errors - * PEM 1 9 - * PKCS#12 1 4 (Started from top) - * X509 2 20 - * PKCS5 2 4 (Started from top) - * DHM 3 11 - * PK 3 15 (Started from top) - * RSA 4 11 - * ECP 4 10 (Started from top) - * MD 5 5 - * HKDF 5 1 (Started from top) - * PKCS7 5 12 (Started from 0x5300) - * SSL 5 3 (Started from 0x5F00) - * CIPHER 6 8 (Started from 0x6080) - * SSL 6 22 (Started from top, plus 0x6000) - * SSL 7 20 (Started from 0x7000, gaps at - * 0x7380, 0x7900-0x7980, 0x7A80-0x7E80) - * - * Module dependent error code (5 bits 0x.00.-0x.F8.) - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** Generic error */ -#define MBEDTLS_ERR_ERROR_GENERIC_ERROR -0x0001 -/** This is a bug in the library */ -#define MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED -0x006E - -/** Hardware accelerator failed */ -#define MBEDTLS_ERR_PLATFORM_HW_ACCEL_FAILED -0x0070 -/** The requested feature is not supported by the platform */ -#define MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED -0x0072 - -/** - * \brief Combines a high-level and low-level error code together. - * - * Wrapper macro for mbedtls_error_add(). See that function for - * more details. - */ -#define MBEDTLS_ERROR_ADD(high, low) \ - mbedtls_error_add(high, low, __FILE__, __LINE__) - -#if defined(MBEDTLS_TEST_HOOKS) -/** - * \brief Testing hook called before adding/combining two error codes together. - * Only used when invasive testing is enabled via MBEDTLS_TEST_HOOKS. - */ -extern void (*mbedtls_test_hook_error_add)(int, int, const char *, int); -#endif - -/** - * \brief Combines a high-level and low-level error code together. - * - * This function can be called directly however it is usually - * called via the #MBEDTLS_ERROR_ADD macro. - * - * While a value of zero is not a negative error code, it is still an - * error code (that denotes success) and can be combined with both a - * negative error code or another value of zero. - * - * \note When invasive testing is enabled via #MBEDTLS_TEST_HOOKS, also try to - * call \link mbedtls_test_hook_error_add \endlink. - * - * \param high high-level error code. See error.h for more details. - * \param low low-level error code. See error.h for more details. - * \param file file where this error code addition occurred. - * \param line line where this error code addition occurred. - */ -static inline int mbedtls_error_add(int high, int low, - const char *file, int line) -{ -#if defined(MBEDTLS_TEST_HOOKS) - if (*mbedtls_test_hook_error_add != NULL) { - (*mbedtls_test_hook_error_add)(high, low, file, line); - } -#endif - (void) file; - (void) line; - - return high + low; -} - -/** - * \brief Translate an Mbed TLS error code into a string representation. - * The result is truncated if necessary and always includes a - * terminating null byte. - * - * \param errnum error code - * \param buffer buffer to place representation in - * \param buflen length of the buffer - */ -void mbedtls_strerror(int errnum, char *buffer, size_t buflen); - -/** - * \brief Translate the high-level part of an Mbed TLS error code into a string - * representation. - * - * This function returns a const pointer to an un-modifiable string. The caller - * must not try to modify the string. It is intended to be used mostly for - * logging purposes. - * - * \param error_code error code - * - * \return The string representation of the error code, or \c NULL if the error - * code is unknown. - */ -const char *mbedtls_high_level_strerr(int error_code); - -/** - * \brief Translate the low-level part of an Mbed TLS error code into a string - * representation. - * - * This function returns a const pointer to an un-modifiable string. The caller - * must not try to modify the string. It is intended to be used mostly for - * logging purposes. - * - * \param error_code error code - * - * \return The string representation of the error code, or \c NULL if the error - * code is unknown. - */ -const char *mbedtls_low_level_strerr(int error_code); - -#ifdef __cplusplus -} -#endif - -#endif /* error.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/gcm.h b/android/mbedtls/arm64-v8a/include/mbedtls/gcm.h deleted file mode 100644 index 390ed4c6d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/gcm.h +++ /dev/null @@ -1,387 +0,0 @@ -/** - * \file gcm.h - * - * \brief This file contains GCM definitions and functions. - * - * The Galois/Counter Mode (GCM) for 128-bit block ciphers is defined - * in D. McGrew, J. Viega, The Galois/Counter Mode of Operation - * (GCM), Natl. Inst. Stand. Technol. - * - * For more information on GCM, see NIST SP 800-38D: Recommendation for - * Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC. - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_GCM_H -#define MBEDTLS_GCM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/cipher.h" - -#if defined(MBEDTLS_BLOCK_CIPHER_C) -#include "mbedtls/block_cipher.h" -#endif - -#include - -#define MBEDTLS_GCM_ENCRYPT 1 -#define MBEDTLS_GCM_DECRYPT 0 - -/** Authenticated decryption failed. */ -#define MBEDTLS_ERR_GCM_AUTH_FAILED -0x0012 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_GCM_BAD_INPUT -0x0014 -/** An output buffer is too small. */ -#define MBEDTLS_ERR_GCM_BUFFER_TOO_SMALL -0x0016 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_GCM_ALT) - -#if defined(MBEDTLS_GCM_LARGE_TABLE) -#define MBEDTLS_GCM_HTABLE_SIZE 256 -#else -#define MBEDTLS_GCM_HTABLE_SIZE 16 -#endif - -/** - * \brief The GCM context structure. - */ -typedef struct mbedtls_gcm_context { -#if defined(MBEDTLS_BLOCK_CIPHER_C) - mbedtls_block_cipher_context_t MBEDTLS_PRIVATE(block_cipher_ctx); /*!< The cipher context used. */ -#else - mbedtls_cipher_context_t MBEDTLS_PRIVATE(cipher_ctx); /*!< The cipher context used. */ -#endif - uint64_t MBEDTLS_PRIVATE(H)[MBEDTLS_GCM_HTABLE_SIZE][2]; /*!< Precalculated HTable. */ - uint64_t MBEDTLS_PRIVATE(len); /*!< The total length of the encrypted data. */ - uint64_t MBEDTLS_PRIVATE(add_len); /*!< The total length of the additional data. */ - unsigned char MBEDTLS_PRIVATE(base_ectr)[16]; /*!< The first ECTR for tag. */ - unsigned char MBEDTLS_PRIVATE(y)[16]; /*!< The Y working value. */ - unsigned char MBEDTLS_PRIVATE(buf)[16]; /*!< The buf working value. */ - unsigned char MBEDTLS_PRIVATE(mode); /*!< The operation to perform: - #MBEDTLS_GCM_ENCRYPT or - #MBEDTLS_GCM_DECRYPT. */ - unsigned char MBEDTLS_PRIVATE(acceleration); /*!< The acceleration to use. */ -} -mbedtls_gcm_context; - -#else /* !MBEDTLS_GCM_ALT */ -#include "gcm_alt.h" -#endif /* !MBEDTLS_GCM_ALT */ - -/** - * \brief This function initializes the specified GCM context, - * to make references valid, and prepares the context - * for mbedtls_gcm_setkey() or mbedtls_gcm_free(). - * - * The function does not bind the GCM context to a particular - * cipher, nor set the key. For this purpose, use - * mbedtls_gcm_setkey(). - * - * \param ctx The GCM context to initialize. This must not be \c NULL. - */ -void mbedtls_gcm_init(mbedtls_gcm_context *ctx); - -/** - * \brief This function associates a GCM context with a - * cipher algorithm and a key. - * - * \param ctx The GCM context. This must be initialized. - * \param cipher The 128-bit block cipher to use. - * \param key The encryption key. This must be a readable buffer of at - * least \p keybits bits. - * \param keybits The key size in bits. Valid options are: - *
  • 128 bits
    • - *
  • 192 bits
    • - *
  • 256 bits
- * - * \return \c 0 on success. - * \return A cipher-specific error code on failure. - */ -int mbedtls_gcm_setkey(mbedtls_gcm_context *ctx, - mbedtls_cipher_id_t cipher, - const unsigned char *key, - unsigned int keybits); - -/** - * \brief This function performs GCM encryption or decryption of a buffer. - * - * \note The output buffer \p output can be the same as the input - * buffer \p input. If \p output is greater than \p input, they - * cannot overlap. - * - * \warning When this function performs a decryption, it outputs the - * authentication tag and does not verify that the data is - * authentic. You should use this function to perform encryption - * only. For decryption, use mbedtls_gcm_auth_decrypt() instead. - * - * \param ctx The GCM context to use for encryption or decryption. This - * must be initialized. - * \param mode The operation to perform: - * - #MBEDTLS_GCM_ENCRYPT to perform authenticated encryption. - * The ciphertext is written to \p output and the - * authentication tag is written to \p tag. - * - #MBEDTLS_GCM_DECRYPT to perform decryption. - * The plaintext is written to \p output and the - * authentication tag is written to \p tag. - * Note that this mode is not recommended, because it does - * not verify the authenticity of the data. For this reason, - * you should use mbedtls_gcm_auth_decrypt() instead of - * calling this function in decryption mode. - * \param length The length of the input data, which is equal to the length - * of the output data. - * \param iv The initialization vector. This must be a readable buffer of - * at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * \param add The buffer holding the additional data. This must be of at - * least that size in Bytes. - * \param add_len The length of the additional data. - * \param input The buffer holding the input data. If \p length is greater - * than zero, this must be a readable buffer of at least that - * size in Bytes. - * \param output The buffer for holding the output data. If \p length is greater - * than zero, this must be a writable buffer of at least that - * size in Bytes. - * \param tag_len The length of the tag to generate. - * \param tag The buffer for holding the tag. This must be a writable - * buffer of at least \p tag_len Bytes. - * - * \return \c 0 if the encryption or decryption was performed - * successfully. Note that in #MBEDTLS_GCM_DECRYPT mode, - * this does not indicate that the data is authentic. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are - * not valid or a cipher-specific error code if the encryption - * or decryption failed. - */ -int mbedtls_gcm_crypt_and_tag(mbedtls_gcm_context *ctx, - int mode, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *input, - unsigned char *output, - size_t tag_len, - unsigned char *tag); - -/** - * \brief This function performs a GCM authenticated decryption of a - * buffer. - * - * \note The output buffer \p output can be the same as the input - * buffer \p input. If \p output is greater than \p input, they - * cannot overlap. Implementations which require - * MBEDTLS_GCM_ALT to be enabled may not provide support for - * overlapping buffers. - * - * \param ctx The GCM context. This must be initialized. - * \param length The length of the ciphertext to decrypt, which is also - * the length of the decrypted plaintext. - * \param iv The initialization vector. This must be a readable buffer - * of at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * \param add The buffer holding the additional data. This must be of at - * least that size in Bytes. - * \param add_len The length of the additional data. - * \param tag The buffer holding the tag to verify. This must be a - * readable buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to verify. - * \param input The buffer holding the ciphertext. If \p length is greater - * than zero, this must be a readable buffer of at least that - * size. - * \param output The buffer for holding the decrypted plaintext. If \p length - * is greater than zero, this must be a writable buffer of at - * least that size. - * - * \return \c 0 if successful and authenticated. - * \return #MBEDTLS_ERR_GCM_AUTH_FAILED if the tag does not match. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT if the lengths or pointers are - * not valid or a cipher-specific error code if the decryption - * failed. - */ -int mbedtls_gcm_auth_decrypt(mbedtls_gcm_context *ctx, - size_t length, - const unsigned char *iv, - size_t iv_len, - const unsigned char *add, - size_t add_len, - const unsigned char *tag, - size_t tag_len, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function starts a GCM encryption or decryption - * operation. - * - * \param ctx The GCM context. This must be initialized. - * \param mode The operation to perform: #MBEDTLS_GCM_ENCRYPT or - * #MBEDTLS_GCM_DECRYPT. - * \param iv The initialization vector. This must be a readable buffer of - * at least \p iv_len Bytes. - * \param iv_len The length of the IV. - * - * \return \c 0 on success. - */ -int mbedtls_gcm_starts(mbedtls_gcm_context *ctx, - int mode, - const unsigned char *iv, - size_t iv_len); - -/** - * \brief This function feeds an input buffer as associated data - * (authenticated but not encrypted data) in a GCM - * encryption or decryption operation. - * - * Call this function after mbedtls_gcm_starts() to pass - * the associated data. If the associated data is empty, - * you do not need to call this function. You may not - * call this function after calling mbedtls_cipher_update(). - * - * \param ctx The GCM context. This must have been started with - * mbedtls_gcm_starts() and must not have yet received - * any input with mbedtls_gcm_update(). - * \param add The buffer holding the additional data, or \c NULL - * if \p add_len is \c 0. - * \param add_len The length of the additional data. If \c 0, - * \p add may be \c NULL. - * - * \return \c 0 on success. - */ -int mbedtls_gcm_update_ad(mbedtls_gcm_context *ctx, - const unsigned char *add, - size_t add_len); - -/** - * \brief This function feeds an input buffer into an ongoing GCM - * encryption or decryption operation. - * - * You may call this function zero, one or more times - * to pass successive parts of the input: the plaintext to - * encrypt, or the ciphertext (not including the tag) to - * decrypt. After the last part of the input, call - * mbedtls_gcm_finish(). - * - * This function may produce output in one of the following - * ways: - * - Immediate output: the output length is always equal - * to the input length. - * - Buffered output: the output consists of a whole number - * of 16-byte blocks. If the total input length so far - * (not including associated data) is 16 \* *B* + *A* - * with *A* < 16 then the total output length is 16 \* *B*. - * - * In particular: - * - It is always correct to call this function with - * \p output_size >= \p input_length + 15. - * - If \p input_length is a multiple of 16 for all the calls - * to this function during an operation, then it is - * correct to use \p output_size = \p input_length. - * - * \note The output buffer \p output can be the same as the input - * buffer \p input. If \p output is greater than \p input, they - * cannot overlap. Implementations which require - * MBEDTLS_GCM_ALT to be enabled may not provide support for - * overlapping buffers. - * - * \param ctx The GCM context. This must be initialized. - * \param input The buffer holding the input data. If \p input_length - * is greater than zero, this must be a readable buffer - * of at least \p input_length bytes. - * \param input_length The length of the input data in bytes. - * \param output The buffer for the output data. If \p output_size - * is greater than zero, this must be a writable buffer of - * of at least \p output_size bytes. - * \param output_size The size of the output buffer in bytes. - * See the function description regarding the output size. - * \param output_length On success, \p *output_length contains the actual - * length of the output written in \p output. - * On failure, the content of \p *output_length is - * unspecified. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure: - * total input length too long, - * unsupported input/output buffer overlap detected, - * or \p output_size too small. - */ -int mbedtls_gcm_update(mbedtls_gcm_context *ctx, - const unsigned char *input, size_t input_length, - unsigned char *output, size_t output_size, - size_t *output_length); - -/** - * \brief This function finishes the GCM operation and generates - * the authentication tag. - * - * It wraps up the GCM stream, and generates the - * tag. The tag can have a maximum length of 16 Bytes. - * - * \param ctx The GCM context. This must be initialized. - * \param tag The buffer for holding the tag. This must be a writable - * buffer of at least \p tag_len Bytes. - * \param tag_len The length of the tag to generate. This must be at least - * four. - * \param output The buffer for the final output. - * If \p output_size is nonzero, this must be a writable - * buffer of at least \p output_size bytes. - * \param output_size The size of the \p output buffer in bytes. - * This must be large enough for the output that - * mbedtls_gcm_update() has not produced. In particular: - * - If mbedtls_gcm_update() produces immediate output, - * or if the total input size is a multiple of \c 16, - * then mbedtls_gcm_finish() never produces any output, - * so \p output_size can be \c 0. - * - \p output_size never needs to be more than \c 15. - * \param output_length On success, \p *output_length contains the actual - * length of the output written in \p output. - * On failure, the content of \p *output_length is - * unspecified. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_GCM_BAD_INPUT on failure: - * invalid value of \p tag_len, - * or \p output_size too small. - */ -int mbedtls_gcm_finish(mbedtls_gcm_context *ctx, - unsigned char *output, size_t output_size, - size_t *output_length, - unsigned char *tag, size_t tag_len); - -/** - * \brief This function clears a GCM context and the underlying - * cipher sub-context. - * - * \param ctx The GCM context to clear. If this is \c NULL, the call has - * no effect. Otherwise, this must be initialized. - */ -void mbedtls_gcm_free(mbedtls_gcm_context *ctx); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The GCM checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_gcm_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - - -#endif /* gcm.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/hkdf.h b/android/mbedtls/arm64-v8a/include/mbedtls/hkdf.h deleted file mode 100644 index 930e93f32..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/hkdf.h +++ /dev/null @@ -1,124 +0,0 @@ -/** - * \file hkdf.h - * - * \brief This file contains the HKDF interface. - * - * The HMAC-based Extract-and-Expand Key Derivation Function (HKDF) is - * specified by RFC 5869. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_HKDF_H -#define MBEDTLS_HKDF_H - -#include "mbedtls/build_info.h" - -#include "mbedtls/md.h" - -/** - * \name HKDF Error codes - * \{ - */ -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_HKDF_BAD_INPUT_DATA -0x5F80 -/** \} name */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief This is the HMAC-based Extract-and-Expand Key Derivation Function - * (HKDF). - * - * \param md A hash function; md.size denotes the length of the hash - * function output in bytes. - * \param salt An optional salt value (a non-secret random value); - * if the salt is not provided, a string of all zeros of - * md.size length is used as the salt. - * \param salt_len The length in bytes of the optional \p salt. - * \param ikm The input keying material. - * \param ikm_len The length in bytes of \p ikm. - * \param info An optional context and application specific information - * string. This can be a zero-length string. - * \param info_len The length of \p info in bytes. - * \param okm The output keying material of \p okm_len bytes. - * \param okm_len The length of the output keying material in bytes. This - * must be less than or equal to 255 * md.size bytes. - * - * \return 0 on success. - * \return #MBEDTLS_ERR_HKDF_BAD_INPUT_DATA when the parameters are invalid. - * \return An MBEDTLS_ERR_MD_* error for errors returned from the underlying - * MD layer. - */ -int mbedtls_hkdf(const mbedtls_md_info_t *md, const unsigned char *salt, - size_t salt_len, const unsigned char *ikm, size_t ikm_len, - const unsigned char *info, size_t info_len, - unsigned char *okm, size_t okm_len); - -/** - * \brief Take the input keying material \p ikm and extract from it a - * fixed-length pseudorandom key \p prk. - * - * \warning This function should only be used if the security of it has been - * studied and established in that particular context (eg. TLS 1.3 - * key schedule). For standard HKDF security guarantees use - * \c mbedtls_hkdf instead. - * - * \param md A hash function; md.size denotes the length of the - * hash function output in bytes. - * \param salt An optional salt value (a non-secret random value); - * if the salt is not provided, a string of all zeros - * of md.size length is used as the salt. - * \param salt_len The length in bytes of the optional \p salt. - * \param ikm The input keying material. - * \param ikm_len The length in bytes of \p ikm. - * \param[out] prk A pseudorandom key of at least md.size bytes. - * - * \return 0 on success. - * \return #MBEDTLS_ERR_HKDF_BAD_INPUT_DATA when the parameters are invalid. - * \return An MBEDTLS_ERR_MD_* error for errors returned from the underlying - * MD layer. - */ -int mbedtls_hkdf_extract(const mbedtls_md_info_t *md, - const unsigned char *salt, size_t salt_len, - const unsigned char *ikm, size_t ikm_len, - unsigned char *prk); - -/** - * \brief Expand the supplied \p prk into several additional pseudorandom - * keys, which is the output of the HKDF. - * - * \warning This function should only be used if the security of it has been - * studied and established in that particular context (eg. TLS 1.3 - * key schedule). For standard HKDF security guarantees use - * \c mbedtls_hkdf instead. - * - * \param md A hash function; md.size denotes the length of the hash - * function output in bytes. - * \param prk A pseudorandom key of at least md.size bytes. \p prk is - * usually the output from the HKDF extract step. - * \param prk_len The length in bytes of \p prk. - * \param info An optional context and application specific information - * string. This can be a zero-length string. - * \param info_len The length of \p info in bytes. - * \param okm The output keying material of \p okm_len bytes. - * \param okm_len The length of the output keying material in bytes. This - * must be less than or equal to 255 * md.size bytes. - * - * \return 0 on success. - * \return #MBEDTLS_ERR_HKDF_BAD_INPUT_DATA when the parameters are invalid. - * \return An MBEDTLS_ERR_MD_* error for errors returned from the underlying - * MD layer. - */ -int mbedtls_hkdf_expand(const mbedtls_md_info_t *md, const unsigned char *prk, - size_t prk_len, const unsigned char *info, - size_t info_len, unsigned char *okm, size_t okm_len); - -#ifdef __cplusplus -} -#endif - -#endif /* hkdf.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/hmac_drbg.h b/android/mbedtls/arm64-v8a/include/mbedtls/hmac_drbg.h deleted file mode 100644 index 18b1b75a6..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/hmac_drbg.h +++ /dev/null @@ -1,434 +0,0 @@ -/** - * \file hmac_drbg.h - * - * \brief The HMAC_DRBG pseudorandom generator. - * - * This module implements the HMAC_DRBG pseudorandom generator described - * in NIST SP 800-90A: Recommendation for Random Number Generation Using - * Deterministic Random Bit Generators. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_HMAC_DRBG_H -#define MBEDTLS_HMAC_DRBG_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/md.h" - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -/* - * Error codes - */ -/** Too many random requested in single call. */ -#define MBEDTLS_ERR_HMAC_DRBG_REQUEST_TOO_BIG -0x0003 -/** Input too large (Entropy + additional). */ -#define MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG -0x0005 -/** Read/write error in file. */ -#define MBEDTLS_ERR_HMAC_DRBG_FILE_IO_ERROR -0x0007 -/** The entropy source failed. */ -#define MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED -0x0009 - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_HMAC_DRBG_RESEED_INTERVAL) -#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ -#endif - -#if !defined(MBEDTLS_HMAC_DRBG_MAX_INPUT) -#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ -#endif - -#if !defined(MBEDTLS_HMAC_DRBG_MAX_REQUEST) -#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -#endif - -#if !defined(MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT) -#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ -#endif - -/** \} name SECTION: Module settings */ - -#define MBEDTLS_HMAC_DRBG_PR_OFF 0 /**< No prediction resistance */ -#define MBEDTLS_HMAC_DRBG_PR_ON 1 /**< Prediction resistance enabled */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * HMAC_DRBG context. - */ -typedef struct mbedtls_hmac_drbg_context { - /* Working state: the key K is not stored explicitly, - * but is implied by the HMAC context */ - mbedtls_md_context_t MBEDTLS_PRIVATE(md_ctx); /*!< HMAC context (inc. K) */ - unsigned char MBEDTLS_PRIVATE(V)[MBEDTLS_MD_MAX_SIZE]; /*!< V in the spec */ - int MBEDTLS_PRIVATE(reseed_counter); /*!< reseed counter */ - - /* Administrative state */ - size_t MBEDTLS_PRIVATE(entropy_len); /*!< entropy bytes grabbed on each (re)seed */ - int MBEDTLS_PRIVATE(prediction_resistance); /*!< enable prediction resistance (Automatic - reseed before every random generation) */ - int MBEDTLS_PRIVATE(reseed_interval); /*!< reseed interval */ - - /* Callbacks */ - int(*MBEDTLS_PRIVATE(f_entropy))(void *, unsigned char *, size_t); /*!< entropy function */ - void *MBEDTLS_PRIVATE(p_entropy); /*!< context for the entropy function */ - -#if defined(MBEDTLS_THREADING_C) - /* Invariant: the mutex is initialized if and only if - * md_ctx->md_info != NULL. This means that the mutex is initialized - * during the initial seeding in mbedtls_hmac_drbg_seed() or - * mbedtls_hmac_drbg_seed_buf() and freed in mbedtls_ctr_drbg_free(). - * - * Note that this invariant may change without notice. Do not rely on it - * and do not access the mutex directly in application code. - */ - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); -#endif -} mbedtls_hmac_drbg_context; - -/** - * \brief HMAC_DRBG context initialization. - * - * This function makes the context ready for mbedtls_hmac_drbg_seed(), - * mbedtls_hmac_drbg_seed_buf() or mbedtls_hmac_drbg_free(). - * - * \note The reseed interval is #MBEDTLS_HMAC_DRBG_RESEED_INTERVAL - * by default. Override this value by calling - * mbedtls_hmac_drbg_set_reseed_interval(). - * - * \param ctx HMAC_DRBG context to be initialized. - */ -void mbedtls_hmac_drbg_init(mbedtls_hmac_drbg_context *ctx); - -/** - * \brief HMAC_DRBG initial seeding. - * - * Set the initial seed and set up the entropy source for future reseeds. - * - * A typical choice for the \p f_entropy and \p p_entropy parameters is - * to use the entropy module: - * - \p f_entropy is mbedtls_entropy_func(); - * - \p p_entropy is an instance of ::mbedtls_entropy_context initialized - * with mbedtls_entropy_init() (which registers the platform's default - * entropy sources). - * - * You can provide a personalization string in addition to the - * entropy source, to make this instantiation as unique as possible. - * - * \note By default, the security strength as defined by NIST is: - * - 128 bits if \p md_info is SHA-1; - * - 192 bits if \p md_info is SHA-224; - * - 256 bits if \p md_info is SHA-256, SHA-384 or SHA-512. - * Note that SHA-256 is just as efficient as SHA-224. - * The security strength can be reduced if a smaller - * entropy length is set with - * mbedtls_hmac_drbg_set_entropy_len(). - * - * \note The default entropy length is the security strength - * (converted from bits to bytes). You can override - * it by calling mbedtls_hmac_drbg_set_entropy_len(). - * - * \note During the initial seeding, this function calls - * the entropy source to obtain a nonce - * whose length is half the entropy length. - */ -#if defined(MBEDTLS_THREADING_C) -/** - * \note When Mbed TLS is built with threading support, - * after this function returns successfully, - * it is safe to call mbedtls_hmac_drbg_random() - * from multiple threads. Other operations, including - * reseeding, are not thread-safe. - */ -#endif /* MBEDTLS_THREADING_C */ -/** - * \param ctx HMAC_DRBG context to be seeded. - * \param md_info MD algorithm to use for HMAC_DRBG. - * \param f_entropy The entropy callback, taking as arguments the - * \p p_entropy context, the buffer to fill, and the - * length of the buffer. - * \p f_entropy is always called with a length that is - * less than or equal to the entropy length. - * \param p_entropy The entropy context to pass to \p f_entropy. - * \param custom The personalization string. - * This can be \c NULL, in which case the personalization - * string is empty regardless of the value of \p len. - * \param len The length of the personalization string. - * This must be at most #MBEDTLS_HMAC_DRBG_MAX_INPUT - * and also at most - * #MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT - \c entropy_len * 3 / 2 - * where \c entropy_len is the entropy length - * described above. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA if \p md_info is - * invalid. - * \return #MBEDTLS_ERR_MD_ALLOC_FAILED if there was not enough - * memory to allocate context data. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED - * if the call to \p f_entropy failed. - */ -int mbedtls_hmac_drbg_seed(mbedtls_hmac_drbg_context *ctx, - const mbedtls_md_info_t *md_info, - int (*f_entropy)(void *, unsigned char *, size_t), - void *p_entropy, - const unsigned char *custom, - size_t len); - -/** - * \brief Initialisation of simplified HMAC_DRBG (never reseeds). - * - * This function is meant for use in algorithms that need a pseudorandom - * input such as deterministic ECDSA. - */ -#if defined(MBEDTLS_THREADING_C) -/** - * \note When Mbed TLS is built with threading support, - * after this function returns successfully, - * it is safe to call mbedtls_hmac_drbg_random() - * from multiple threads. Other operations, including - * reseeding, are not thread-safe. - */ -#endif /* MBEDTLS_THREADING_C */ -/** - * \param ctx HMAC_DRBG context to be initialised. - * \param md_info MD algorithm to use for HMAC_DRBG. - * \param data Concatenation of the initial entropy string and - * the additional data. - * \param data_len Length of \p data in bytes. - * - * \return \c 0 if successful. or - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA if \p md_info is - * invalid. - * \return #MBEDTLS_ERR_MD_ALLOC_FAILED if there was not enough - * memory to allocate context data. - */ -int mbedtls_hmac_drbg_seed_buf(mbedtls_hmac_drbg_context *ctx, - const mbedtls_md_info_t *md_info, - const unsigned char *data, size_t data_len); - -/** - * \brief This function turns prediction resistance on or off. - * The default value is off. - * - * \note If enabled, entropy is gathered at the beginning of - * every call to mbedtls_hmac_drbg_random_with_add() - * or mbedtls_hmac_drbg_random(). - * Only use this if your entropy source has sufficient - * throughput. - * - * \param ctx The HMAC_DRBG context. - * \param resistance #MBEDTLS_HMAC_DRBG_PR_ON or #MBEDTLS_HMAC_DRBG_PR_OFF. - */ -void mbedtls_hmac_drbg_set_prediction_resistance(mbedtls_hmac_drbg_context *ctx, - int resistance); - -/** - * \brief This function sets the amount of entropy grabbed on each - * seed or reseed. - * - * See the documentation of mbedtls_hmac_drbg_seed() for the default value. - * - * \param ctx The HMAC_DRBG context. - * \param len The amount of entropy to grab, in bytes. - */ -void mbedtls_hmac_drbg_set_entropy_len(mbedtls_hmac_drbg_context *ctx, - size_t len); - -/** - * \brief Set the reseed interval. - * - * The reseed interval is the number of calls to mbedtls_hmac_drbg_random() - * or mbedtls_hmac_drbg_random_with_add() after which the entropy function - * is called again. - * - * The default value is #MBEDTLS_HMAC_DRBG_RESEED_INTERVAL. - * - * \param ctx The HMAC_DRBG context. - * \param interval The reseed interval. - */ -void mbedtls_hmac_drbg_set_reseed_interval(mbedtls_hmac_drbg_context *ctx, - int interval); - -/** - * \brief This function updates the state of the HMAC_DRBG context. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param ctx The HMAC_DRBG context. - * \param additional The data to update the state with. - * If this is \c NULL, there is no additional data. - * \param add_len Length of \p additional in bytes. - * Unused if \p additional is \c NULL. - * - * \return \c 0 on success, or an error from the underlying - * hash calculation. - */ -int mbedtls_hmac_drbg_update(mbedtls_hmac_drbg_context *ctx, - const unsigned char *additional, size_t add_len); - -/** - * \brief This function reseeds the HMAC_DRBG context, that is - * extracts data from the entropy source. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param ctx The HMAC_DRBG context. - * \param additional Additional data to add to the state. - * If this is \c NULL, there is no additional data - * and \p len should be \c 0. - * \param len The length of the additional data. - * This must be at most #MBEDTLS_HMAC_DRBG_MAX_INPUT - * and also at most - * #MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT - \c entropy_len - * where \c entropy_len is the entropy length - * (see mbedtls_hmac_drbg_set_entropy_len()). - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED - * if a call to the entropy function failed. - */ -int mbedtls_hmac_drbg_reseed(mbedtls_hmac_drbg_context *ctx, - const unsigned char *additional, size_t len); - -/** - * \brief This function updates an HMAC_DRBG instance with additional - * data and uses it to generate random data. - * - * This function automatically reseeds if the reseed counter is exceeded - * or prediction resistance is enabled. - * - * \note This function is not thread-safe. It is not safe - * to call this function if another thread might be - * concurrently obtaining random numbers from the same - * context or updating or reseeding the same context. - * - * \param p_rng The HMAC_DRBG context. This must be a pointer to a - * #mbedtls_hmac_drbg_context structure. - * \param output The buffer to fill. - * \param output_len The length of the buffer in bytes. - * This must be at most #MBEDTLS_HMAC_DRBG_MAX_REQUEST. - * \param additional Additional data to update with. - * If this is \c NULL, there is no additional data - * and \p add_len should be \c 0. - * \param add_len The length of the additional data. - * This must be at most #MBEDTLS_HMAC_DRBG_MAX_INPUT. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED - * if a call to the entropy source failed. - * \return #MBEDTLS_ERR_HMAC_DRBG_REQUEST_TOO_BIG if - * \p output_len > #MBEDTLS_HMAC_DRBG_MAX_REQUEST. - * \return #MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG if - * \p add_len > #MBEDTLS_HMAC_DRBG_MAX_INPUT. - */ -int mbedtls_hmac_drbg_random_with_add(void *p_rng, - unsigned char *output, size_t output_len, - const unsigned char *additional, - size_t add_len); - -/** - * \brief This function uses HMAC_DRBG to generate random data. - * - * This function automatically reseeds if the reseed counter is exceeded - * or prediction resistance is enabled. - */ -#if defined(MBEDTLS_THREADING_C) -/** - * \note When Mbed TLS is built with threading support, - * it is safe to call mbedtls_ctr_drbg_random() - * from multiple threads. Other operations, including - * reseeding, are not thread-safe. - */ -#endif /* MBEDTLS_THREADING_C */ -/** - * \param p_rng The HMAC_DRBG context. This must be a pointer to a - * #mbedtls_hmac_drbg_context structure. - * \param output The buffer to fill. - * \param out_len The length of the buffer in bytes. - * This must be at most #MBEDTLS_HMAC_DRBG_MAX_REQUEST. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED - * if a call to the entropy source failed. - * \return #MBEDTLS_ERR_HMAC_DRBG_REQUEST_TOO_BIG if - * \p out_len > #MBEDTLS_HMAC_DRBG_MAX_REQUEST. - */ -int mbedtls_hmac_drbg_random(void *p_rng, unsigned char *output, size_t out_len); - -/** - * \brief This function resets HMAC_DRBG context to the state immediately - * after initial call of mbedtls_hmac_drbg_init(). - * - * \param ctx The HMAC_DRBG context to free. - */ -void mbedtls_hmac_drbg_free(mbedtls_hmac_drbg_context *ctx); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief This function writes a seed file. - * - * \param ctx The HMAC_DRBG context. - * \param path The name of the file. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_HMAC_DRBG_FILE_IO_ERROR on file error. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED on reseed - * failure. - */ -int mbedtls_hmac_drbg_write_seed_file(mbedtls_hmac_drbg_context *ctx, const char *path); - -/** - * \brief This function reads and updates a seed file. The seed - * is added to this instance. - * - * \param ctx The HMAC_DRBG context. - * \param path The name of the file. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_HMAC_DRBG_FILE_IO_ERROR on file error. - * \return #MBEDTLS_ERR_HMAC_DRBG_ENTROPY_SOURCE_FAILED on - * reseed failure. - * \return #MBEDTLS_ERR_HMAC_DRBG_INPUT_TOO_BIG if the existing - * seed file is too large. - */ -int mbedtls_hmac_drbg_update_seed_file(mbedtls_hmac_drbg_context *ctx, const char *path); -#endif /* MBEDTLS_FS_IO */ - - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief The HMAC_DRBG Checkup routine. - * - * \return \c 0 if successful. - * \return \c 1 if the test failed. - */ -int mbedtls_hmac_drbg_self_test(int verbose); -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* hmac_drbg.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/lms.h b/android/mbedtls/arm64-v8a/include/mbedtls/lms.h deleted file mode 100644 index 2f297914b..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/lms.h +++ /dev/null @@ -1,440 +0,0 @@ -/** - * \file lms.h - * - * \brief This file provides an API for the LMS post-quantum-safe stateful-hash - public-key signature scheme as defined in RFC8554 and NIST.SP.200-208. - * This implementation currently only supports a single parameter set - * MBEDTLS_LMS_SHA256_M32_H10 in order to reduce complexity. This is one - * of the signature schemes recommended by the IETF draft SUIT standard - * for IOT firmware upgrades (RFC9019). - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_LMS_H -#define MBEDTLS_LMS_H - -#include -#include - -#include "mbedtls/private_access.h" -#include "mbedtls/build_info.h" - -#define MBEDTLS_ERR_LMS_BAD_INPUT_DATA -0x0011 /**< Bad data has been input to an LMS function */ -#define MBEDTLS_ERR_LMS_OUT_OF_PRIVATE_KEYS -0x0013 /**< Specified LMS key has utilised all of its private keys */ -#define MBEDTLS_ERR_LMS_VERIFY_FAILED -0x0015 /**< LMS signature verification failed */ -#define MBEDTLS_ERR_LMS_ALLOC_FAILED -0x0017 /**< LMS failed to allocate space for a private key */ -#define MBEDTLS_ERR_LMS_BUFFER_TOO_SMALL -0x0019 /**< Input/output buffer is too small to contain requited data */ - -/* Currently only defined for SHA256, 32 is the max hash output size */ -#define MBEDTLS_LMOTS_N_HASH_LEN_MAX (32u) -#define MBEDTLS_LMOTS_P_SIG_DIGIT_COUNT_MAX (34u) -#define MBEDTLS_LMOTS_N_HASH_LEN(type) ((type) == MBEDTLS_LMOTS_SHA256_N32_W8 ? 32u : 0) -#define MBEDTLS_LMOTS_I_KEY_ID_LEN (16u) -#define MBEDTLS_LMOTS_Q_LEAF_ID_LEN (4u) -#define MBEDTLS_LMOTS_TYPE_LEN (4u) -#define MBEDTLS_LMOTS_P_SIG_DIGIT_COUNT(type) ((type) == MBEDTLS_LMOTS_SHA256_N32_W8 ? 34u : 0) -#define MBEDTLS_LMOTS_C_RANDOM_VALUE_LEN(type) (MBEDTLS_LMOTS_N_HASH_LEN(type)) - -#define MBEDTLS_LMOTS_SIG_LEN(type) (MBEDTLS_LMOTS_TYPE_LEN + \ - MBEDTLS_LMOTS_C_RANDOM_VALUE_LEN(type) + \ - (MBEDTLS_LMOTS_P_SIG_DIGIT_COUNT(type) * \ - MBEDTLS_LMOTS_N_HASH_LEN(type))) - - -#define MBEDTLS_LMS_TYPE_LEN (4) -#define MBEDTLS_LMS_H_TREE_HEIGHT(type) ((type) == MBEDTLS_LMS_SHA256_M32_H10 ? 10u : 0) - -/* The length of a hash output, Currently only implemented for SHA256. - * Max is 32 bytes. - */ -#define MBEDTLS_LMS_M_NODE_BYTES(type) ((type) == MBEDTLS_LMS_SHA256_M32_H10 ? 32 : 0) -#define MBEDTLS_LMS_M_NODE_BYTES_MAX 32 - -#define MBEDTLS_LMS_SIG_LEN(type, otstype) (MBEDTLS_LMOTS_Q_LEAF_ID_LEN + \ - MBEDTLS_LMOTS_SIG_LEN(otstype) + \ - MBEDTLS_LMS_TYPE_LEN + \ - (MBEDTLS_LMS_H_TREE_HEIGHT(type) * \ - MBEDTLS_LMS_M_NODE_BYTES(type))) - -#define MBEDTLS_LMS_PUBLIC_KEY_LEN(type) (MBEDTLS_LMS_TYPE_LEN + \ - MBEDTLS_LMOTS_TYPE_LEN + \ - MBEDTLS_LMOTS_I_KEY_ID_LEN + \ - MBEDTLS_LMS_M_NODE_BYTES(type)) - - -#ifdef __cplusplus -extern "C" { -#endif - -/** The Identifier of the LMS parameter set, as per - * https://www.iana.org/assignments/leighton-micali-signatures/leighton-micali-signatures.xhtml - * We are only implementing a subset of the types, particularly H10, for the sake of simplicity. - */ -typedef enum { - MBEDTLS_LMS_SHA256_M32_H10 = 0x6, -} mbedtls_lms_algorithm_type_t; - -/** The Identifier of the LMOTS parameter set, as per - * https://www.iana.org/assignments/leighton-micali-signatures/leighton-micali-signatures.xhtml. - * We are only implementing a subset of the types, particularly N32_W8, for the sake of simplicity. - */ -typedef enum { - MBEDTLS_LMOTS_SHA256_N32_W8 = 4 -} mbedtls_lmots_algorithm_type_t; - -/** LMOTS parameters structure. - * - * This contains the metadata associated with an LMOTS key, detailing the - * algorithm type, the key ID, and the leaf identifier should be key be part of - * a LMS key. - */ -typedef struct { - unsigned char MBEDTLS_PRIVATE(I_key_identifier[MBEDTLS_LMOTS_I_KEY_ID_LEN]); /*!< The key - identifier. */ - unsigned char MBEDTLS_PRIVATE(q_leaf_identifier[MBEDTLS_LMOTS_Q_LEAF_ID_LEN]); /*!< Which - leaf of the LMS key this is. - 0 if the key is not part of an LMS key. */ - mbedtls_lmots_algorithm_type_t MBEDTLS_PRIVATE(type); /*!< The LM-OTS key type identifier as - per IANA. Only SHA256_N32_W8 is - currently supported. */ -} mbedtls_lmots_parameters_t; - -/** LMOTS public context structure. - * - * A LMOTS public key is a hash output, and the applicable parameter set. - * - * The context must be initialized before it is used. A public key must either - * be imported or generated from a private context. - * - * \dot - * digraph lmots_public_t { - * UNINITIALIZED -> INIT [label="init"]; - * HAVE_PUBLIC_KEY -> INIT [label="free"]; - * INIT -> HAVE_PUBLIC_KEY [label="import_public_key"]; - * INIT -> HAVE_PUBLIC_KEY [label="calculate_public_key from private key"]; - * HAVE_PUBLIC_KEY -> HAVE_PUBLIC_KEY [label="export_public_key"]; - * } - * \enddot - */ -typedef struct { - mbedtls_lmots_parameters_t MBEDTLS_PRIVATE(params); - unsigned char MBEDTLS_PRIVATE(public_key)[MBEDTLS_LMOTS_N_HASH_LEN_MAX]; - unsigned char MBEDTLS_PRIVATE(have_public_key); /*!< Whether the context contains a public key. - Boolean values only. */ -} mbedtls_lmots_public_t; - -#if defined(MBEDTLS_LMS_PRIVATE) -/** LMOTS private context structure. - * - * A LMOTS private key is one hash output for each of digit of the digest + - * checksum, and the applicable parameter set. - * - * The context must be initialized before it is used. A public key must either - * be imported or generated from a private context. - * - * \dot - * digraph lmots_public_t { - * UNINITIALIZED -> INIT [label="init"]; - * HAVE_PRIVATE_KEY -> INIT [label="free"]; - * INIT -> HAVE_PRIVATE_KEY [label="generate_private_key"]; - * HAVE_PRIVATE_KEY -> INIT [label="sign"]; - * } - * \enddot - */ -typedef struct { - mbedtls_lmots_parameters_t MBEDTLS_PRIVATE(params); - unsigned char MBEDTLS_PRIVATE(private_key)[MBEDTLS_LMOTS_P_SIG_DIGIT_COUNT_MAX][ - MBEDTLS_LMOTS_N_HASH_LEN_MAX]; - unsigned char MBEDTLS_PRIVATE(have_private_key); /*!< Whether the context contains a private key. - Boolean values only. */ -} mbedtls_lmots_private_t; -#endif /* defined(MBEDTLS_LMS_PRIVATE) */ - - -/** LMS parameters structure. - * - * This contains the metadata associated with an LMS key, detailing the - * algorithm type, the type of the underlying OTS algorithm, and the key ID. - */ -typedef struct { - unsigned char MBEDTLS_PRIVATE(I_key_identifier[MBEDTLS_LMOTS_I_KEY_ID_LEN]); /*!< The key - identifier. */ - mbedtls_lmots_algorithm_type_t MBEDTLS_PRIVATE(otstype); /*!< The LM-OTS key type identifier as - per IANA. Only SHA256_N32_W8 is - currently supported. */ - mbedtls_lms_algorithm_type_t MBEDTLS_PRIVATE(type); /*!< The LMS key type identifier as per - IANA. Only SHA256_M32_H10 is currently - supported. */ -} mbedtls_lms_parameters_t; - -/** LMS public context structure. - * - * A LMS public key is the hash output that is the root of the Merkle tree, and - * the applicable parameter set - * - * The context must be initialized before it is used. A public key must either - * be imported or generated from a private context. - * - * \dot - * digraph lms_public_t { - * UNINITIALIZED -> INIT [label="init"]; - * HAVE_PUBLIC_KEY -> INIT [label="free"]; - * INIT -> HAVE_PUBLIC_KEY [label="import_public_key"]; - * INIT -> HAVE_PUBLIC_KEY [label="calculate_public_key from private key"]; - * HAVE_PUBLIC_KEY -> HAVE_PUBLIC_KEY [label="export_public_key"]; - * } - * \enddot - */ -typedef struct { - mbedtls_lms_parameters_t MBEDTLS_PRIVATE(params); - unsigned char MBEDTLS_PRIVATE(T_1_pub_key)[MBEDTLS_LMS_M_NODE_BYTES_MAX]; /*!< The public key, in - the form of the Merkle tree root node. */ - unsigned char MBEDTLS_PRIVATE(have_public_key); /*!< Whether the context contains a public key. - Boolean values only. */ -} mbedtls_lms_public_t; - - -#if defined(MBEDTLS_LMS_PRIVATE) -/** LMS private context structure. - * - * A LMS private key is a set of LMOTS private keys, an index to the next usable - * key, and the applicable parameter set. - * - * The context must be initialized before it is used. A public key must either - * be imported or generated from a private context. - * - * \dot - * digraph lms_public_t { - * UNINITIALIZED -> INIT [label="init"]; - * HAVE_PRIVATE_KEY -> INIT [label="free"]; - * INIT -> HAVE_PRIVATE_KEY [label="generate_private_key"]; - * } - * \enddot - */ -typedef struct { - mbedtls_lms_parameters_t MBEDTLS_PRIVATE(params); - uint32_t MBEDTLS_PRIVATE(q_next_usable_key); /*!< The index of the next OTS key that has not - been used. */ - mbedtls_lmots_private_t *MBEDTLS_PRIVATE(ots_private_keys); /*!< The private key material. One OTS key - for each leaf node in the Merkle tree. NULL - when have_private_key is 0 and non-NULL otherwise. - is 2^MBEDTLS_LMS_H_TREE_HEIGHT(type) in length. */ - mbedtls_lmots_public_t *MBEDTLS_PRIVATE(ots_public_keys); /*!< The OTS key public keys, used to - build the Merkle tree. NULL - when have_private_key is 0 and - non-NULL otherwise. - Is 2^MBEDTLS_LMS_H_TREE_HEIGHT(type) - in length. */ - unsigned char MBEDTLS_PRIVATE(have_private_key); /*!< Whether the context contains a private key. - Boolean values only. */ -} mbedtls_lms_private_t; -#endif /* defined(MBEDTLS_LMS_PRIVATE) */ - -/** - * \brief This function initializes an LMS public context - * - * \param ctx The uninitialized LMS context that will then be - * initialized. - */ -void mbedtls_lms_public_init(mbedtls_lms_public_t *ctx); - -/** - * \brief This function uninitializes an LMS public context - * - * \param ctx The initialized LMS context that will then be - * uninitialized. - */ -void mbedtls_lms_public_free(mbedtls_lms_public_t *ctx); - -/** - * \brief This function imports an LMS public key into a - * public LMS context. - * - * \note Before this function is called, the context must - * have been initialized. - * - * \note See IETF RFC8554 for details of the encoding of - * this public key. - * - * \param ctx The initialized LMS context store the key in. - * \param key The buffer from which the key will be read. - * #MBEDTLS_LMS_PUBLIC_KEY_LEN bytes will be read from - * this. - * \param key_size The size of the key being imported. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_import_public_key(mbedtls_lms_public_t *ctx, - const unsigned char *key, size_t key_size); - -/** - * \brief This function exports an LMS public key from a - * LMS public context that already contains a public - * key. - * - * \note Before this function is called, the context must - * have been initialized and the context must contain - * a public key. - * - * \note See IETF RFC8554 for details of the encoding of - * this public key. - * - * \param ctx The initialized LMS public context that contains - * the public key. - * \param key The buffer into which the key will be output. Must - * be at least #MBEDTLS_LMS_PUBLIC_KEY_LEN in size. - * \param key_size The size of the key buffer. - * \param key_len If not NULL, will be written with the size of the - * key. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_export_public_key(const mbedtls_lms_public_t *ctx, - unsigned char *key, size_t key_size, - size_t *key_len); - -/** - * \brief This function verifies a LMS signature, using a - * LMS context that contains a public key. - * - * \note Before this function is called, the context must - * have been initialized and must contain a public key - * (either by import or generation). - * - * \param ctx The initialized LMS public context from which the - * public key will be read. - * \param msg The buffer from which the message will be read. - * \param msg_size The size of the message that will be read. - * \param sig The buf from which the signature will be read. - * #MBEDTLS_LMS_SIG_LEN bytes will be read from - * this. - * \param sig_size The size of the signature to be verified. - * - * \return \c 0 on successful verification. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_verify(const mbedtls_lms_public_t *ctx, - const unsigned char *msg, size_t msg_size, - const unsigned char *sig, size_t sig_size); - -#if defined(MBEDTLS_LMS_PRIVATE) -/** - * \brief This function initializes an LMS private context - * - * \param ctx The uninitialized LMS private context that will - * then be initialized. */ -void mbedtls_lms_private_init(mbedtls_lms_private_t *ctx); - -/** - * \brief This function uninitializes an LMS private context - * - * \param ctx The initialized LMS private context that will then - * be uninitialized. - */ -void mbedtls_lms_private_free(mbedtls_lms_private_t *ctx); - -/** - * \brief This function generates an LMS private key, and - * stores in into an LMS private context. - * - * \warning This function is **not intended for use in - * production**, due to as-yet unsolved problems with - * handling stateful keys. The API for this function - * may change considerably in future versions. - * - * \note The seed must have at least 256 bits of entropy. - * - * \param ctx The initialized LMOTS context to generate the key - * into. - * \param type The LMS parameter set identifier. - * \param otstype The LMOTS parameter set identifier. - * \param f_rng The RNG function to be used to generate the key ID. - * \param p_rng The RNG context to be passed to f_rng - * \param seed The seed used to deterministically generate the - * key. - * \param seed_size The length of the seed. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_generate_private_key(mbedtls_lms_private_t *ctx, - mbedtls_lms_algorithm_type_t type, - mbedtls_lmots_algorithm_type_t otstype, - mbedtls_f_rng_t *f_rng, - void *p_rng, const unsigned char *seed, - size_t seed_size); - -/** - * \brief This function calculates an LMS public key from a - * LMS context that already contains a private key. - * - * \note Before this function is called, the context must - * have been initialized and the context must contain - * a private key. - * - * \param ctx The initialized LMS public context to calculate the key - * from and store it into. - * - * \param priv_ctx The LMS private context to read the private key - * from. This must have been initialized and contain a - * private key. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_calculate_public_key(mbedtls_lms_public_t *ctx, - const mbedtls_lms_private_t *priv_ctx); - -/** - * \brief This function creates a LMS signature, using a - * LMS context that contains unused private keys. - * - * \warning This function is **not intended for use in - * production**, due to as-yet unsolved problems with - * handling stateful keys. The API for this function - * may change considerably in future versions. - * - * \note Before this function is called, the context must - * have been initialized and must contain a private - * key. - * - * \note Each of the LMOTS private keys inside a LMS private - * key can only be used once. If they are reused, then - * attackers may be able to forge signatures with that - * key. This is all handled transparently, but it is - * important to not perform copy operations on LMS - * contexts that contain private key material. - * - * \param ctx The initialized LMS private context from which the - * private key will be read. - * \param f_rng The RNG function to be used for signature - * generation. - * \param p_rng The RNG context to be passed to f_rng - * \param msg The buffer from which the message will be read. - * \param msg_size The size of the message that will be read. - * \param sig The buf into which the signature will be stored. - * Must be at least #MBEDTLS_LMS_SIG_LEN in size. - * \param sig_size The size of the buffer the signature will be - * written into. - * \param sig_len If not NULL, will be written with the size of the - * signature. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_lms_sign(mbedtls_lms_private_t *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, const unsigned char *msg, - unsigned int msg_size, unsigned char *sig, size_t sig_size, - size_t *sig_len); -#endif /* defined(MBEDTLS_LMS_PRIVATE) */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_LMS_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/mbedtls_config.h b/android/mbedtls/arm64-v8a/include/mbedtls/mbedtls_config.h deleted file mode 100644 index 75eff2d89..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/mbedtls_config.h +++ /dev/null @@ -1,4392 +0,0 @@ -/** - * \file mbedtls_config.h - * - * \brief Configuration options (set of defines) - * - * This set of compile-time options may be used to enable - * or disable features selectively, and reduce the global - * memory footprint. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -/** - * This is an optional version symbol that enables compatibility handling of - * config files. - * - * It is equal to the #MBEDTLS_VERSION_NUMBER of the Mbed TLS version that - * introduced the config format we want to be compatible with. - */ -//#define MBEDTLS_CONFIG_VERSION 0x03000000 - -/** - * \name SECTION: System support - * - * This section sets system specific settings. - * \{ - */ - -/** - * \def MBEDTLS_HAVE_ASM - * - * The compiler has support for asm(). - * - * Requires support for asm() in compiler. - * - * Used in: - * library/aesni.h - * library/aria.c - * library/bn_mul.h - * library/constant_time.c - * library/padlock.h - * - * Required by: - * MBEDTLS_AESCE_C - * MBEDTLS_AESNI_C (on some platforms) - * MBEDTLS_PADLOCK_C - * - * Comment to disable the use of assembly code. - */ -#define MBEDTLS_HAVE_ASM - -/** - * \def MBEDTLS_NO_UDBL_DIVISION - * - * The platform lacks support for double-width integer division (64-bit - * division on a 32-bit platform, 128-bit division on a 64-bit platform). - * - * Used in: - * include/mbedtls/bignum.h - * library/bignum.c - * - * The bignum code uses double-width division to speed up some operations. - * Double-width division is often implemented in software that needs to - * be linked with the program. The presence of a double-width integer - * type is usually detected automatically through preprocessor macros, - * but the automatic detection cannot know whether the code needs to - * and can be linked with an implementation of division for that type. - * By default division is assumed to be usable if the type is present. - * Uncomment this option to prevent the use of double-width division. - * - * Note that division for the native integer type is always required. - * Furthermore, a 64-bit type is always required even on a 32-bit - * platform, but it need not support multiplication or division. In some - * cases it is also desirable to disable some double-width operations. For - * example, if double-width division is implemented in software, disabling - * it can reduce code size in some embedded targets. - */ -//#define MBEDTLS_NO_UDBL_DIVISION - -/** - * \def MBEDTLS_NO_64BIT_MULTIPLICATION - * - * The platform lacks support for 32x32 -> 64-bit multiplication. - * - * Used in: - * library/poly1305.c - * - * Some parts of the library may use multiplication of two unsigned 32-bit - * operands with a 64-bit result in order to speed up computations. On some - * platforms, this is not available in hardware and has to be implemented in - * software, usually in a library provided by the toolchain. - * - * Sometimes it is not desirable to have to link to that library. This option - * removes the dependency of that library on platforms that lack a hardware - * 64-bit multiplier by embedding a software implementation in Mbed TLS. - * - * Note that depending on the compiler, this may decrease performance compared - * to using the library function provided by the toolchain. - */ -//#define MBEDTLS_NO_64BIT_MULTIPLICATION - -/** - * \def MBEDTLS_HAVE_SSE2 - * - * CPU supports SSE2 instruction set. - * - * Uncomment if the CPU supports SSE2 (IA-32 specific). - */ -//#define MBEDTLS_HAVE_SSE2 - -/** - * \def MBEDTLS_HAVE_TIME - * - * System has time.h and time(). - * The time does not need to be correct, only time differences are used, - * by contrast with MBEDTLS_HAVE_TIME_DATE - * - * Defining MBEDTLS_HAVE_TIME allows you to specify MBEDTLS_PLATFORM_TIME_ALT, - * MBEDTLS_PLATFORM_TIME_MACRO, MBEDTLS_PLATFORM_TIME_TYPE_MACRO and - * MBEDTLS_PLATFORM_STD_TIME. - * - * Comment if your system does not support time functions. - * - * \note If MBEDTLS_TIMING_C is set - to enable the semi-portable timing - * interface - timing.c will include time.h on suitable platforms - * regardless of the setting of MBEDTLS_HAVE_TIME, unless - * MBEDTLS_TIMING_ALT is used. See timing.c for more information. - */ -#define MBEDTLS_HAVE_TIME - -/** - * \def MBEDTLS_HAVE_TIME_DATE - * - * System has time.h, time(), and an implementation for - * mbedtls_platform_gmtime_r() (see below). - * The time needs to be correct (not necessarily very accurate, but at least - * the date should be correct). This is used to verify the validity period of - * X.509 certificates. - * - * Comment if your system does not have a correct clock. - * - * \note mbedtls_platform_gmtime_r() is an abstraction in platform_util.h that - * behaves similarly to the gmtime_r() function from the C standard. Refer to - * the documentation for mbedtls_platform_gmtime_r() for more information. - * - * \note It is possible to configure an implementation for - * mbedtls_platform_gmtime_r() at compile-time by using the macro - * MBEDTLS_PLATFORM_GMTIME_R_ALT. - */ -#define MBEDTLS_HAVE_TIME_DATE - -/** - * \def MBEDTLS_PLATFORM_MEMORY - * - * Enable the memory allocation layer. - * - * By default Mbed TLS uses the system-provided calloc() and free(). - * This allows different allocators (self-implemented or provided) to be - * provided to the platform abstraction layer. - * - * Enabling #MBEDTLS_PLATFORM_MEMORY without the - * MBEDTLS_PLATFORM_{FREE,CALLOC}_MACROs will provide - * "mbedtls_platform_set_calloc_free()" allowing you to set an alternative calloc() and - * free() function pointer at runtime. - * - * Enabling #MBEDTLS_PLATFORM_MEMORY and specifying - * MBEDTLS_PLATFORM_{CALLOC,FREE}_MACROs will allow you to specify the - * alternate function at compile time. - * - * An overview of how the value of mbedtls_calloc is determined: - * - * - if !MBEDTLS_PLATFORM_MEMORY - * - mbedtls_calloc = calloc - * - if MBEDTLS_PLATFORM_MEMORY - * - if (MBEDTLS_PLATFORM_CALLOC_MACRO && MBEDTLS_PLATFORM_FREE_MACRO): - * - mbedtls_calloc = MBEDTLS_PLATFORM_CALLOC_MACRO - * - if !(MBEDTLS_PLATFORM_CALLOC_MACRO && MBEDTLS_PLATFORM_FREE_MACRO): - * - Dynamic setup via mbedtls_platform_set_calloc_free is now possible with a default value MBEDTLS_PLATFORM_STD_CALLOC. - * - How is MBEDTLS_PLATFORM_STD_CALLOC handled? - * - if MBEDTLS_PLATFORM_NO_STD_FUNCTIONS: - * - MBEDTLS_PLATFORM_STD_CALLOC is not set to anything; - * - MBEDTLS_PLATFORM_STD_MEM_HDR can be included if present; - * - if !MBEDTLS_PLATFORM_NO_STD_FUNCTIONS: - * - if MBEDTLS_PLATFORM_STD_CALLOC is present: - * - User-defined MBEDTLS_PLATFORM_STD_CALLOC is respected; - * - if !MBEDTLS_PLATFORM_STD_CALLOC: - * - MBEDTLS_PLATFORM_STD_CALLOC = calloc - * - * - At this point the presence of MBEDTLS_PLATFORM_STD_CALLOC is checked. - * - if !MBEDTLS_PLATFORM_STD_CALLOC - * - MBEDTLS_PLATFORM_STD_CALLOC = uninitialized_calloc - * - * - mbedtls_calloc = MBEDTLS_PLATFORM_STD_CALLOC. - * - * Defining MBEDTLS_PLATFORM_CALLOC_MACRO and #MBEDTLS_PLATFORM_STD_CALLOC at the same time is not possible. - * MBEDTLS_PLATFORM_CALLOC_MACRO and MBEDTLS_PLATFORM_FREE_MACRO must both be defined or undefined at the same time. - * #MBEDTLS_PLATFORM_STD_CALLOC and #MBEDTLS_PLATFORM_STD_FREE do not have to be defined at the same time, as, if they are used, - * dynamic setup of these functions is possible. See the tree above to see how are they handled in all cases. - * An uninitialized #MBEDTLS_PLATFORM_STD_CALLOC always fails, returning a null pointer. - * An uninitialized #MBEDTLS_PLATFORM_STD_FREE does not do anything. - * - * Requires: MBEDTLS_PLATFORM_C - * - * Enable this layer to allow use of alternative memory allocators. - */ -//#define MBEDTLS_PLATFORM_MEMORY - -/** - * \def MBEDTLS_PLATFORM_NO_STD_FUNCTIONS - * - * Do not assign standard functions in the platform layer (e.g. calloc() to - * MBEDTLS_PLATFORM_STD_CALLOC and printf() to MBEDTLS_PLATFORM_STD_PRINTF) - * - * This makes sure there are no linking errors on platforms that do not support - * these functions. You will HAVE to provide alternatives, either at runtime - * via the platform_set_xxx() functions or at compile time by setting - * the MBEDTLS_PLATFORM_STD_XXX defines, or enabling a - * MBEDTLS_PLATFORM_XXX_MACRO. - * - * Requires: MBEDTLS_PLATFORM_C - * - * Uncomment to prevent default assignment of standard functions in the - * platform layer. - */ -//#define MBEDTLS_PLATFORM_NO_STD_FUNCTIONS - -/** - * \def MBEDTLS_PLATFORM_EXIT_ALT - * - * MBEDTLS_PLATFORM_XXX_ALT: Uncomment a macro to let Mbed TLS support the - * function in the platform abstraction layer. - * - * Example: In case you uncomment MBEDTLS_PLATFORM_PRINTF_ALT, Mbed TLS will - * provide a function "mbedtls_platform_set_printf()" that allows you to set an - * alternative printf function pointer. - * - * All these define require MBEDTLS_PLATFORM_C to be defined! - * - * \note MBEDTLS_PLATFORM_SNPRINTF_ALT is required on Windows; - * it will be enabled automatically by check_config.h - * - * \warning MBEDTLS_PLATFORM_XXX_ALT cannot be defined at the same time as - * MBEDTLS_PLATFORM_XXX_MACRO! - * - * Requires: MBEDTLS_PLATFORM_TIME_ALT requires MBEDTLS_HAVE_TIME - * - * Uncomment a macro to enable alternate implementation of specific base - * platform function - */ -//#define MBEDTLS_PLATFORM_SETBUF_ALT -//#define MBEDTLS_PLATFORM_EXIT_ALT -//#define MBEDTLS_PLATFORM_TIME_ALT -//#define MBEDTLS_PLATFORM_FPRINTF_ALT -//#define MBEDTLS_PLATFORM_PRINTF_ALT -//#define MBEDTLS_PLATFORM_SNPRINTF_ALT -//#define MBEDTLS_PLATFORM_VSNPRINTF_ALT -//#define MBEDTLS_PLATFORM_NV_SEED_ALT -//#define MBEDTLS_PLATFORM_SETUP_TEARDOWN_ALT -//#define MBEDTLS_PLATFORM_MS_TIME_ALT - -/** - * Uncomment the macro to let Mbed TLS use your alternate implementation of - * mbedtls_platform_gmtime_r(). This replaces the default implementation in - * platform_util.c. - * - * gmtime() is not a thread-safe function as defined in the C standard. The - * library will try to use safer implementations of this function, such as - * gmtime_r() when available. However, if Mbed TLS cannot identify the target - * system, the implementation of mbedtls_platform_gmtime_r() will default to - * using the standard gmtime(). In this case, calls from the library to - * gmtime() will be guarded by the global mutex mbedtls_threading_gmtime_mutex - * if MBEDTLS_THREADING_C is enabled. We recommend that calls from outside the - * library are also guarded with this mutex to avoid race conditions. However, - * if the macro MBEDTLS_PLATFORM_GMTIME_R_ALT is defined, Mbed TLS will - * unconditionally use the implementation for mbedtls_platform_gmtime_r() - * supplied at compile time. - */ -//#define MBEDTLS_PLATFORM_GMTIME_R_ALT - -/** - * Uncomment the macro to let Mbed TLS use your alternate implementation of - * mbedtls_platform_zeroize(), to wipe sensitive data in memory. This replaces - * the default implementation in platform_util.c. - * - * By default, the library uses a system function such as memset_s() - * (optional feature of C11), explicit_bzero() (BSD and compatible), or - * SecureZeroMemory (Windows). If no such function is detected, the library - * falls back to a plain C implementation. Compilers are technically - * permitted to optimize this implementation out, meaning that the memory is - * not actually wiped. The library tries to prevent that, but the C language - * makes it impossible to guarantee that the memory will always be wiped. - * - * If your platform provides a guaranteed method to wipe memory which - * `platform_util.c` does not detect, define this macro to the name of - * a function that takes two arguments, a `void *` pointer and a length, - * and wipes that many bytes starting at the specified address. For example, - * if your platform has explicit_bzero() but `platform_util.c` does not - * detect its presence, define `MBEDTLS_PLATFORM_ZEROIZE_ALT` to be - * `explicit_bzero` to use that function as mbedtls_platform_zeroize(). - */ -//#define MBEDTLS_PLATFORM_ZEROIZE_ALT - -/** - * \def MBEDTLS_DEPRECATED_WARNING - * - * Mark deprecated functions and features so that they generate a warning if - * used. Functionality deprecated in one version will usually be removed in the - * next version. You can enable this to help you prepare the transition to a - * new major version by making sure your code is not using this functionality. - * - * This only works with GCC and Clang. With other compilers, you may want to - * use MBEDTLS_DEPRECATED_REMOVED - * - * Uncomment to get warnings on using deprecated functions and features. - */ -//#define MBEDTLS_DEPRECATED_WARNING - -/** - * \def MBEDTLS_DEPRECATED_REMOVED - * - * Remove deprecated functions and features so that they generate an error if - * used. Functionality deprecated in one version will usually be removed in the - * next version. You can enable this to help you prepare the transition to a - * new major version by making sure your code is not using this functionality. - * - * Uncomment to get errors on using deprecated functions and features. - */ -//#define MBEDTLS_DEPRECATED_REMOVED - -/** \} name SECTION: System support */ - -/** - * \name SECTION: Mbed TLS feature support - * - * This section sets support for features that are or are not needed - * within the modules that are enabled. - * \{ - */ - -/** - * \def MBEDTLS_TIMING_ALT - * - * Uncomment to provide your own alternate implementation for - * mbedtls_timing_get_timer(), mbedtls_set_alarm(), mbedtls_set/get_delay() - * - * Only works if you have MBEDTLS_TIMING_C enabled. - * - * You will need to provide a header "timing_alt.h" and an implementation at - * compile time. - */ -//#define MBEDTLS_TIMING_ALT - -/** - * \def MBEDTLS_AES_ALT - * - * MBEDTLS__MODULE_NAME__ALT: Uncomment a macro to let Mbed TLS use your - * alternate core implementation of a symmetric crypto, an arithmetic or hash - * module (e.g. platform specific assembly optimized implementations). Keep - * in mind that the function prototypes should remain the same. - * - * This replaces the whole module. If you only want to replace one of the - * functions, use one of the MBEDTLS__FUNCTION_NAME__ALT flags. - * - * Example: In case you uncomment MBEDTLS_AES_ALT, Mbed TLS will no longer - * provide the "struct mbedtls_aes_context" definition and omit the base - * function declarations and implementations. "aes_alt.h" will be included from - * "aes.h" to include the new function definitions. - * - * Uncomment a macro to enable alternate implementation of the corresponding - * module. - * - * \warning MD5, DES and SHA-1 are considered weak and their - * use constitutes a security risk. If possible, we recommend - * avoiding dependencies on them, and considering stronger message - * digests and ciphers instead. - * - */ -//#define MBEDTLS_AES_ALT -//#define MBEDTLS_ARIA_ALT -//#define MBEDTLS_CAMELLIA_ALT -//#define MBEDTLS_CCM_ALT -//#define MBEDTLS_CHACHA20_ALT -//#define MBEDTLS_CHACHAPOLY_ALT -//#define MBEDTLS_CMAC_ALT -//#define MBEDTLS_DES_ALT -//#define MBEDTLS_DHM_ALT -//#define MBEDTLS_ECJPAKE_ALT -//#define MBEDTLS_GCM_ALT -//#define MBEDTLS_NIST_KW_ALT -//#define MBEDTLS_MD5_ALT -//#define MBEDTLS_POLY1305_ALT -//#define MBEDTLS_RIPEMD160_ALT -//#define MBEDTLS_RSA_ALT -//#define MBEDTLS_SHA1_ALT -//#define MBEDTLS_SHA256_ALT -//#define MBEDTLS_SHA512_ALT - -/* - * When replacing the elliptic curve module, please consider, that it is - * implemented with two .c files: - * - ecp.c - * - ecp_curves.c - * You can replace them very much like all the other MBEDTLS__MODULE_NAME__ALT - * macros as described above. The only difference is that you have to make sure - * that you provide functionality for both .c files. - */ -//#define MBEDTLS_ECP_ALT - -/** - * \def MBEDTLS_SHA256_PROCESS_ALT - * - * MBEDTLS__FUNCTION_NAME__ALT: Uncomment a macro to let Mbed TLS use you - * alternate core implementation of symmetric crypto or hash function. Keep in - * mind that function prototypes should remain the same. - * - * This replaces only one function. The header file from Mbed TLS is still - * used, in contrast to the MBEDTLS__MODULE_NAME__ALT flags. - * - * Example: In case you uncomment MBEDTLS_SHA256_PROCESS_ALT, Mbed TLS will - * no longer provide the mbedtls_sha1_process() function, but it will still provide - * the other function (using your mbedtls_sha1_process() function) and the definition - * of mbedtls_sha1_context, so your implementation of mbedtls_sha1_process must be compatible - * with this definition. - * - * \note If you use the AES_xxx_ALT macros, then it is recommended to also set - * MBEDTLS_AES_ROM_TABLES in order to help the linker garbage-collect the AES - * tables. - * - * Uncomment a macro to enable alternate implementation of the corresponding - * function. - * - * \warning MD5, DES and SHA-1 are considered weak and their use - * constitutes a security risk. If possible, we recommend avoiding - * dependencies on them, and considering stronger message digests - * and ciphers instead. - * - * \warning If both MBEDTLS_ECDSA_SIGN_ALT and MBEDTLS_ECDSA_DETERMINISTIC are - * enabled, then the deterministic ECDH signature functions pass the - * the static HMAC-DRBG as RNG to mbedtls_ecdsa_sign(). Therefore - * alternative implementations should use the RNG only for generating - * the ephemeral key and nothing else. If this is not possible, then - * MBEDTLS_ECDSA_DETERMINISTIC should be disabled and an alternative - * implementation should be provided for mbedtls_ecdsa_sign_det_ext(). - * - */ -//#define MBEDTLS_MD5_PROCESS_ALT -//#define MBEDTLS_RIPEMD160_PROCESS_ALT -//#define MBEDTLS_SHA1_PROCESS_ALT -//#define MBEDTLS_SHA256_PROCESS_ALT -//#define MBEDTLS_SHA512_PROCESS_ALT -//#define MBEDTLS_DES_SETKEY_ALT -//#define MBEDTLS_DES_CRYPT_ECB_ALT -//#define MBEDTLS_DES3_CRYPT_ECB_ALT -//#define MBEDTLS_AES_SETKEY_ENC_ALT -//#define MBEDTLS_AES_SETKEY_DEC_ALT -//#define MBEDTLS_AES_ENCRYPT_ALT -//#define MBEDTLS_AES_DECRYPT_ALT -//#define MBEDTLS_ECDH_GEN_PUBLIC_ALT -//#define MBEDTLS_ECDH_COMPUTE_SHARED_ALT -//#define MBEDTLS_ECDSA_VERIFY_ALT -//#define MBEDTLS_ECDSA_SIGN_ALT -//#define MBEDTLS_ECDSA_GENKEY_ALT - -/** - * \def MBEDTLS_ECP_INTERNAL_ALT - * - * Expose a part of the internal interface of the Elliptic Curve Point module. - * - * MBEDTLS_ECP__FUNCTION_NAME__ALT: Uncomment a macro to let Mbed TLS use your - * alternative core implementation of elliptic curve arithmetic. Keep in mind - * that function prototypes should remain the same. - * - * This partially replaces one function. The header file from Mbed TLS is still - * used, in contrast to the MBEDTLS_ECP_ALT flag. The original implementation - * is still present and it is used for group structures not supported by the - * alternative. - * - * The original implementation can in addition be removed by setting the - * MBEDTLS_ECP_NO_FALLBACK option, in which case any function for which the - * corresponding MBEDTLS_ECP__FUNCTION_NAME__ALT macro is defined will not be - * able to fallback to curves not supported by the alternative implementation. - * - * Any of these options become available by defining MBEDTLS_ECP_INTERNAL_ALT - * and implementing the following functions: - * unsigned char mbedtls_internal_ecp_grp_capable( - * const mbedtls_ecp_group *grp ) - * int mbedtls_internal_ecp_init( const mbedtls_ecp_group *grp ) - * void mbedtls_internal_ecp_free( const mbedtls_ecp_group *grp ) - * The mbedtls_internal_ecp_grp_capable function should return 1 if the - * replacement functions implement arithmetic for the given group and 0 - * otherwise. - * The functions mbedtls_internal_ecp_init and mbedtls_internal_ecp_free are - * called before and after each point operation and provide an opportunity to - * implement optimized set up and tear down instructions. - * - * Example: In case you set MBEDTLS_ECP_INTERNAL_ALT and - * MBEDTLS_ECP_DOUBLE_JAC_ALT, Mbed TLS will still provide the ecp_double_jac() - * function, but will use your mbedtls_internal_ecp_double_jac() if the group - * for the operation is supported by your implementation (i.e. your - * mbedtls_internal_ecp_grp_capable() function returns 1 for this group). If the - * group is not supported by your implementation, then the original Mbed TLS - * implementation of ecp_double_jac() is used instead, unless this fallback - * behaviour is disabled by setting MBEDTLS_ECP_NO_FALLBACK (in which case - * ecp_double_jac() will return MBEDTLS_ERR_ECP_FEATURE_UNAVAILABLE). - * - * The function prototypes and the definition of mbedtls_ecp_group and - * mbedtls_ecp_point will not change based on MBEDTLS_ECP_INTERNAL_ALT, so your - * implementation of mbedtls_internal_ecp__function_name__ must be compatible - * with their definitions. - * - * Uncomment a macro to enable alternate implementation of the corresponding - * function. - */ -/* Required for all the functions in this section */ -//#define MBEDTLS_ECP_INTERNAL_ALT -/* Turn off software fallback for curves not supported in hardware */ -//#define MBEDTLS_ECP_NO_FALLBACK -/* Support for Weierstrass curves with Jacobi representation */ -//#define MBEDTLS_ECP_RANDOMIZE_JAC_ALT -//#define MBEDTLS_ECP_ADD_MIXED_ALT -//#define MBEDTLS_ECP_DOUBLE_JAC_ALT -//#define MBEDTLS_ECP_NORMALIZE_JAC_MANY_ALT -//#define MBEDTLS_ECP_NORMALIZE_JAC_ALT -/* Support for curves with Montgomery arithmetic */ -//#define MBEDTLS_ECP_DOUBLE_ADD_MXZ_ALT -//#define MBEDTLS_ECP_RANDOMIZE_MXZ_ALT -//#define MBEDTLS_ECP_NORMALIZE_MXZ_ALT - -/** - * \def MBEDTLS_ENTROPY_HARDWARE_ALT - * - * Uncomment this macro to let Mbed TLS use your own implementation of a - * hardware entropy collector. - * - * Your function must be called \c mbedtls_hardware_poll(), have the same - * prototype as declared in library/entropy_poll.h, and accept NULL as first - * argument. - * - * Uncomment to use your own hardware entropy collector. - */ -//#define MBEDTLS_ENTROPY_HARDWARE_ALT - -/** - * \def MBEDTLS_AES_ROM_TABLES - * - * Use precomputed AES tables stored in ROM. - * - * Uncomment this macro to use precomputed AES tables stored in ROM. - * Comment this macro to generate AES tables in RAM at runtime. - * - * Tradeoff: Using precomputed ROM tables reduces RAM usage by ~8kb - * (or ~2kb if \c MBEDTLS_AES_FEWER_TABLES is used) and reduces the - * initialization time before the first AES operation can be performed. - * It comes at the cost of additional ~8kb ROM use (resp. ~2kb if \c - * MBEDTLS_AES_FEWER_TABLES below is used), and potentially degraded - * performance if ROM access is slower than RAM access. - * - * This option is independent of \c MBEDTLS_AES_FEWER_TABLES. - */ -//#define MBEDTLS_AES_ROM_TABLES - -/** - * \def MBEDTLS_AES_FEWER_TABLES - * - * Use less ROM/RAM for AES tables. - * - * Uncommenting this macro omits 75% of the AES tables from - * ROM / RAM (depending on the value of \c MBEDTLS_AES_ROM_TABLES) - * by computing their values on the fly during operations - * (the tables are entry-wise rotations of one another). - * - * Tradeoff: Uncommenting this reduces the RAM / ROM footprint - * by ~6kb but at the cost of more arithmetic operations during - * runtime. Specifically, one has to compare 4 accesses within - * different tables to 4 accesses with additional arithmetic - * operations within the same table. The performance gain/loss - * depends on the system and memory details. - * - * This option is independent of \c MBEDTLS_AES_ROM_TABLES. - */ -//#define MBEDTLS_AES_FEWER_TABLES - -/** - * \def MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH - * - * Use only 128-bit keys in AES operations to save ROM. - * - * Uncomment this macro to remove support for AES operations that use 192- - * or 256-bit keys. - * - * Uncommenting this macro reduces the size of AES code by ~300 bytes - * on v8-M/Thumb2. - * - * Module: library/aes.c - * - * Requires: MBEDTLS_AES_C - */ -//#define MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH - -/* - * Disable plain C implementation for AES. - * - * When the plain C implementation is enabled, and an implementation using a - * special CPU feature (such as MBEDTLS_AESCE_C) is also enabled, runtime - * detection will be used to select between them. - * - * If only one implementation is present, runtime detection will not be used. - * This configuration will crash at runtime if running on a CPU without the - * necessary features. It will not build unless at least one of MBEDTLS_AESCE_C - * and/or MBEDTLS_AESNI_C is enabled & present in the build. - */ -//#define MBEDTLS_AES_USE_HARDWARE_ONLY - -/** - * \def MBEDTLS_CAMELLIA_SMALL_MEMORY - * - * Use less ROM for the Camellia implementation (saves about 768 bytes). - * - * Uncomment this macro to use less memory for Camellia. - */ -//#define MBEDTLS_CAMELLIA_SMALL_MEMORY - -/** - * \def MBEDTLS_CHECK_RETURN_WARNING - * - * If this macro is defined, emit a compile-time warning if application code - * calls a function without checking its return value, but the return value - * should generally be checked in portable applications. - * - * This is only supported on platforms where #MBEDTLS_CHECK_RETURN is - * implemented. Otherwise this option has no effect. - * - * Uncomment to get warnings on using fallible functions without checking - * their return value. - * - * \note This feature is a work in progress. - * Warnings will be added to more functions in the future. - * - * \note A few functions are considered critical, and ignoring the return - * value of these functions will trigger a warning even if this - * macro is not defined. To completely disable return value check - * warnings, define #MBEDTLS_CHECK_RETURN with an empty expansion. - */ -//#define MBEDTLS_CHECK_RETURN_WARNING - -/** - * \def MBEDTLS_CIPHER_MODE_CBC - * - * Enable Cipher Block Chaining mode (CBC) for symmetric ciphers. - */ -#define MBEDTLS_CIPHER_MODE_CBC - -/** - * \def MBEDTLS_CIPHER_MODE_CFB - * - * Enable Cipher Feedback mode (CFB) for symmetric ciphers. - */ -#define MBEDTLS_CIPHER_MODE_CFB - -/** - * \def MBEDTLS_CIPHER_MODE_CTR - * - * Enable Counter Block Cipher mode (CTR) for symmetric ciphers. - */ -#define MBEDTLS_CIPHER_MODE_CTR - -/** - * \def MBEDTLS_CIPHER_MODE_OFB - * - * Enable Output Feedback mode (OFB) for symmetric ciphers. - */ -#define MBEDTLS_CIPHER_MODE_OFB - -/** - * \def MBEDTLS_CIPHER_MODE_XTS - * - * Enable Xor-encrypt-xor with ciphertext stealing mode (XTS) for AES. - */ -#define MBEDTLS_CIPHER_MODE_XTS - -/** - * \def MBEDTLS_CIPHER_NULL_CIPHER - * - * Enable NULL cipher. - * Warning: Only do so when you know what you are doing. This allows for - * encryption or channels without any security! - * - * To enable the following ciphersuites: - * MBEDTLS_TLS_ECDH_ECDSA_WITH_NULL_SHA - * MBEDTLS_TLS_ECDH_RSA_WITH_NULL_SHA - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_NULL_SHA - * MBEDTLS_TLS_ECDHE_RSA_WITH_NULL_SHA - * MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA - * MBEDTLS_TLS_RSA_WITH_NULL_SHA256 - * MBEDTLS_TLS_RSA_WITH_NULL_SHA - * MBEDTLS_TLS_RSA_WITH_NULL_MD5 - * MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA - * MBEDTLS_TLS_PSK_WITH_NULL_SHA384 - * MBEDTLS_TLS_PSK_WITH_NULL_SHA256 - * MBEDTLS_TLS_PSK_WITH_NULL_SHA - * - * Uncomment this macro to enable the NULL cipher and ciphersuites - */ -//#define MBEDTLS_CIPHER_NULL_CIPHER - -/** - * \def MBEDTLS_CIPHER_PADDING_PKCS7 - * - * MBEDTLS_CIPHER_PADDING_XXX: Uncomment or comment macros to add support for - * specific padding modes in the cipher layer with cipher modes that support - * padding (e.g. CBC) - * - * If you disable all padding modes, only full blocks can be used with CBC. - * - * Enable padding modes in the cipher layer. - */ -#define MBEDTLS_CIPHER_PADDING_PKCS7 -#define MBEDTLS_CIPHER_PADDING_ONE_AND_ZEROS -#define MBEDTLS_CIPHER_PADDING_ZEROS_AND_LEN -#define MBEDTLS_CIPHER_PADDING_ZEROS - -/** \def MBEDTLS_CTR_DRBG_USE_128_BIT_KEY - * - * Uncomment this macro to use a 128-bit key in the CTR_DRBG module. - * Without this, CTR_DRBG uses a 256-bit key - * unless \c MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH is set. - */ -//#define MBEDTLS_CTR_DRBG_USE_128_BIT_KEY - -/** - * Enable the verified implementations of ECDH primitives from Project Everest - * (currently only Curve25519). This feature changes the layout of ECDH - * contexts and therefore is a compatibility break for applications that access - * fields of a mbedtls_ecdh_context structure directly. See also - * MBEDTLS_ECDH_LEGACY_CONTEXT in include/mbedtls/ecdh.h. - * - * The Everest code is provided under the Apache 2.0 license only; therefore enabling this - * option is not compatible with taking the library under the GPL v2.0-or-later license. - */ -//#define MBEDTLS_ECDH_VARIANT_EVEREST_ENABLED - -/** - * \def MBEDTLS_ECP_DP_SECP192R1_ENABLED - * - * MBEDTLS_ECP_XXXX_ENABLED: Enables specific curves within the Elliptic Curve - * module. By default all supported curves are enabled. - * - * Comment macros to disable the curve and functions for it - */ -/* Short Weierstrass curves (supporting ECP, ECDH, ECDSA) */ -#define MBEDTLS_ECP_DP_SECP192R1_ENABLED -#define MBEDTLS_ECP_DP_SECP224R1_ENABLED -#define MBEDTLS_ECP_DP_SECP256R1_ENABLED -#define MBEDTLS_ECP_DP_SECP384R1_ENABLED -#define MBEDTLS_ECP_DP_SECP521R1_ENABLED -#define MBEDTLS_ECP_DP_SECP192K1_ENABLED -#define MBEDTLS_ECP_DP_SECP224K1_ENABLED -#define MBEDTLS_ECP_DP_SECP256K1_ENABLED -#define MBEDTLS_ECP_DP_BP256R1_ENABLED -#define MBEDTLS_ECP_DP_BP384R1_ENABLED -#define MBEDTLS_ECP_DP_BP512R1_ENABLED -/* Montgomery curves (supporting ECP) */ -#define MBEDTLS_ECP_DP_CURVE25519_ENABLED -#define MBEDTLS_ECP_DP_CURVE448_ENABLED - -/** - * \def MBEDTLS_ECP_NIST_OPTIM - * - * Enable specific 'modulo p' routines for each NIST prime. - * Depending on the prime and architecture, makes operations 4 to 8 times - * faster on the corresponding curve. - * - * Comment this macro to disable NIST curves optimisation. - */ -#define MBEDTLS_ECP_NIST_OPTIM - -/** - * \def MBEDTLS_ECP_RESTARTABLE - * - * Enable "non-blocking" ECC operations that can return early and be resumed. - * - * This allows various functions to pause by returning - * #MBEDTLS_ERR_ECP_IN_PROGRESS (or, for functions in the SSL module, - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS) and then be called later again in - * order to further progress and eventually complete their operation. This is - * controlled through mbedtls_ecp_set_max_ops() which limits the maximum - * number of ECC operations a function may perform before pausing; see - * mbedtls_ecp_set_max_ops() for more information. - * - * This is useful in non-threaded environments if you want to avoid blocking - * for too long on ECC (and, hence, X.509 or SSL/TLS) operations. - * - * This option: - * - Adds xxx_restartable() variants of existing operations in the - * following modules, with corresponding restart context types: - * - ECP (for Short Weierstrass curves only): scalar multiplication (mul), - * linear combination (muladd); - * - ECDSA: signature generation & verification; - * - PK: signature generation & verification; - * - X509: certificate chain verification. - * - Adds mbedtls_ecdh_enable_restart() in the ECDH module. - * - Changes the behaviour of TLS 1.2 clients (not servers) when using the - * ECDHE-ECDSA key exchange (not other key exchanges) to make all ECC - * computations restartable: - * - ECDH operations from the key exchange, only for Short Weierstrass - * curves, only when MBEDTLS_USE_PSA_CRYPTO is not enabled. - * - verification of the server's key exchange signature; - * - verification of the server's certificate chain; - * - generation of the client's signature if client authentication is used, - * with an ECC key/certificate. - * - * \note In the cases above, the usual SSL/TLS functions, such as - * mbedtls_ssl_handshake(), can now return - * MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS. - * - * \note When this option and MBEDTLS_USE_PSA_CRYPTO are both enabled, - * restartable operations in PK, X.509 and TLS (see above) are not - * using PSA. On the other hand, ECDH computations in TLS are using - * PSA, and are not restartable. These are temporary limitations that - * should be lifted in the future. - * - * \note This option only works with the default software implementation of - * elliptic curve functionality. It is incompatible with - * MBEDTLS_ECP_ALT, MBEDTLS_ECDH_XXX_ALT, MBEDTLS_ECDSA_XXX_ALT. - * - * Requires: MBEDTLS_ECP_C - * - * Uncomment this macro to enable restartable ECC computations. - */ -//#define MBEDTLS_ECP_RESTARTABLE - -/** - * Uncomment to enable using new bignum code in the ECC modules. - * - * \warning This is currently experimental, incomplete and therefore should not - * be used in production. - */ -//#define MBEDTLS_ECP_WITH_MPI_UINT - -/** - * \def MBEDTLS_ECDSA_DETERMINISTIC - * - * Enable deterministic ECDSA (RFC 6979). - * Standard ECDSA is "fragile" in the sense that lack of entropy when signing - * may result in a compromise of the long-term signing key. This is avoided by - * the deterministic variant. - * - * Requires: MBEDTLS_HMAC_DRBG_C, MBEDTLS_ECDSA_C - * - * Comment this macro to disable deterministic ECDSA. - */ -#define MBEDTLS_ECDSA_DETERMINISTIC - -/** - * \def MBEDTLS_KEY_EXCHANGE_PSK_ENABLED - * - * Enable the PSK based ciphersuite modes in SSL / TLS. - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_KEY_EXCHANGE_PSK_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED - * - * Enable the DHE-PSK based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_DHM_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 - * - * \warning Using DHE constitutes a security risk as it - * is not possible to validate custom DH parameters. - * If possible, it is recommended users should consider - * preferring other methods of key exchange. - * See dhm.h for more details. - * - */ -#define MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED - * - * Enable the ECDHE-PSK based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_ECDH_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDH) - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED - * - * Enable the RSA-PSK based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_RSA_C, MBEDTLS_PKCS1_V15, - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_RSA_ENABLED - * - * Enable the RSA-only based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_RSA_C, MBEDTLS_PKCS1_V15, - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA - */ -#define MBEDTLS_KEY_EXCHANGE_RSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED - * - * Enable the DHE-RSA based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_DHM_C, MBEDTLS_RSA_C, MBEDTLS_PKCS1_V15, - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA - * - * \warning Using DHE constitutes a security risk as it - * is not possible to validate custom DH parameters. - * If possible, it is recommended users should consider - * preferring other methods of key exchange. - * See dhm.h for more details. - * - */ -#define MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED - * - * Enable the ECDHE-RSA based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_ECDH_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDH) - * MBEDTLS_RSA_C - * MBEDTLS_PKCS1_V15 - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED - * - * Enable the ECDHE-ECDSA based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_ECDH_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDH) - * MBEDTLS_ECDSA_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDSA) - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED - * - * Enable the ECDH-ECDSA based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_ECDH_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDH) - * MBEDTLS_ECDSA_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDSA) - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 - */ -#define MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED - * - * Enable the ECDH-RSA based ciphersuite modes in SSL / TLS. - * - * Requires: MBEDTLS_ECDH_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDH) - * MBEDTLS_RSA_C - * MBEDTLS_X509_CRT_PARSE_C - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 - */ -#define MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED - -/** - * \def MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED - * - * Enable the ECJPAKE based ciphersuite modes in SSL / TLS. - * - * \warning This is currently experimental. EC J-PAKE support is based on the - * Thread v1.0.0 specification; incompatible changes to the specification - * might still happen. For this reason, this is disabled by default. - * - * Requires: MBEDTLS_ECJPAKE_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_JPAKE) - * SHA-256 (via MBEDTLS_SHA256_C or a PSA driver) - * MBEDTLS_ECP_DP_SECP256R1_ENABLED - * - * \warning If SHA-256 is provided only by a PSA driver, you must call - * psa_crypto_init() before the first handshake (even if - * MBEDTLS_USE_PSA_CRYPTO is disabled). - * - * This enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECJPAKE_WITH_AES_128_CCM_8 - */ -//#define MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED - -/** - * \def MBEDTLS_PK_PARSE_EC_EXTENDED - * - * Enhance support for reading EC keys using variants of SEC1 not allowed by - * RFC 5915 and RFC 5480. - * - * Currently this means parsing the SpecifiedECDomain choice of EC - * parameters (only known groups are supported, not arbitrary domains, to - * avoid validation issues). - * - * Disable if you only need to support RFC 5915 + 5480 key formats. - */ -#define MBEDTLS_PK_PARSE_EC_EXTENDED - -/** - * \def MBEDTLS_PK_PARSE_EC_COMPRESSED - * - * Enable the support for parsing public keys of type Short Weierstrass - * (MBEDTLS_ECP_DP_SECP_XXX and MBEDTLS_ECP_DP_BP_XXX) which are using the - * compressed point format. This parsing is done through ECP module's functions. - * - * \note As explained in the description of MBEDTLS_ECP_PF_COMPRESSED (in ecp.h) - * the only unsupported curves are MBEDTLS_ECP_DP_SECP224R1 and - * MBEDTLS_ECP_DP_SECP224K1. - */ -#define MBEDTLS_PK_PARSE_EC_COMPRESSED - -/** - * \def MBEDTLS_ERROR_STRERROR_DUMMY - * - * Enable a dummy error function to make use of mbedtls_strerror() in - * third party libraries easier when MBEDTLS_ERROR_C is disabled - * (no effect when MBEDTLS_ERROR_C is enabled). - * - * You can safely disable this if MBEDTLS_ERROR_C is enabled, or if you're - * not using mbedtls_strerror() or error_strerror() in your application. - * - * Disable if you run into name conflicts and want to really remove the - * mbedtls_strerror() - */ -#define MBEDTLS_ERROR_STRERROR_DUMMY - -/** - * \def MBEDTLS_GENPRIME - * - * Enable the prime-number generation code. - * - * Requires: MBEDTLS_BIGNUM_C - */ -#define MBEDTLS_GENPRIME - -/** - * \def MBEDTLS_FS_IO - * - * Enable functions that use the filesystem. - */ -#define MBEDTLS_FS_IO - -/** - * \def MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES - * - * Do not add default entropy sources in mbedtls_entropy_init(). - * - * This is useful to have more control over the added entropy sources in an - * application. - * - * Uncomment this macro to prevent loading of default entropy functions. - */ -//#define MBEDTLS_NO_DEFAULT_ENTROPY_SOURCES - -/** - * \def MBEDTLS_NO_PLATFORM_ENTROPY - * - * Do not use built-in platform entropy functions. - * This is useful if your platform does not support - * standards like the /dev/urandom or Windows CryptoAPI. - * - * Uncomment this macro to disable the built-in platform entropy functions. - */ -//#define MBEDTLS_NO_PLATFORM_ENTROPY - -/** - * \def MBEDTLS_ENTROPY_FORCE_SHA256 - * - * Force the entropy accumulator to use a SHA-256 accumulator instead of the - * default SHA-512 based one (if both are available). - * - * Requires: MBEDTLS_SHA256_C - * - * On 32-bit systems SHA-256 can be much faster than SHA-512. Use this option - * if you have performance concerns. - * - * This option is only useful if both MBEDTLS_SHA256_C and - * MBEDTLS_SHA512_C are defined. Otherwise the available hash module is used. - */ -//#define MBEDTLS_ENTROPY_FORCE_SHA256 - -/** - * \def MBEDTLS_ENTROPY_NV_SEED - * - * Enable the non-volatile (NV) seed file-based entropy source. - * (Also enables the NV seed read/write functions in the platform layer) - * - * This is crucial (if not required) on systems that do not have a - * cryptographic entropy source (in hardware or kernel) available. - * - * Requires: MBEDTLS_ENTROPY_C, MBEDTLS_PLATFORM_C - * - * \note The read/write functions that are used by the entropy source are - * determined in the platform layer, and can be modified at runtime and/or - * compile-time depending on the flags (MBEDTLS_PLATFORM_NV_SEED_*) used. - * - * \note If you use the default implementation functions that read a seedfile - * with regular fopen(), please make sure you make a seedfile with the - * proper name (defined in MBEDTLS_PLATFORM_STD_NV_SEED_FILE) and at - * least MBEDTLS_ENTROPY_BLOCK_SIZE bytes in size that can be read from - * and written to or you will get an entropy source error! The default - * implementation will only use the first MBEDTLS_ENTROPY_BLOCK_SIZE - * bytes from the file. - * - * \note The entropy collector will write to the seed file before entropy is - * given to an external source, to update it. - */ -//#define MBEDTLS_ENTROPY_NV_SEED - -/* MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER - * - * Enable key identifiers that encode a key owner identifier. - * - * The owner of a key is identified by a value of type ::mbedtls_key_owner_id_t - * which is currently hard-coded to be int32_t. - * - * Note that this option is meant for internal use only and may be removed - * without notice. - */ -//#define MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER - -/** - * \def MBEDTLS_MEMORY_DEBUG - * - * Enable debugging of buffer allocator memory issues. Automatically prints - * (to stderr) all (fatal) messages on memory allocation issues. Enables - * function for 'debug output' of allocated memory. - * - * Requires: MBEDTLS_MEMORY_BUFFER_ALLOC_C - * - * Uncomment this macro to let the buffer allocator print out error messages. - */ -//#define MBEDTLS_MEMORY_DEBUG - -/** - * \def MBEDTLS_MEMORY_BACKTRACE - * - * Include backtrace information with each allocated block. - * - * Requires: MBEDTLS_MEMORY_BUFFER_ALLOC_C - * GLIBC-compatible backtrace() and backtrace_symbols() support - * - * Uncomment this macro to include backtrace information - */ -//#define MBEDTLS_MEMORY_BACKTRACE - -/** - * \def MBEDTLS_PK_RSA_ALT_SUPPORT - * - * Support external private RSA keys (eg from a HSM) in the PK layer. - * - * Comment this macro to disable support for external private RSA keys. - */ -#define MBEDTLS_PK_RSA_ALT_SUPPORT - -/** - * \def MBEDTLS_PKCS1_V15 - * - * Enable support for PKCS#1 v1.5 encoding. - * - * Requires: MBEDTLS_RSA_C - * - * This enables support for PKCS#1 v1.5 operations. - */ -#define MBEDTLS_PKCS1_V15 - -/** - * \def MBEDTLS_PKCS1_V21 - * - * Enable support for PKCS#1 v2.1 encoding. - * - * Requires: MBEDTLS_RSA_C - * - * \warning If using a hash that is only provided by PSA drivers, you must - * call psa_crypto_init() before doing any PKCS#1 v2.1 operation. - * - * This enables support for RSAES-OAEP and RSASSA-PSS operations. - */ -#define MBEDTLS_PKCS1_V21 - -/** \def MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS - * - * Enable support for platform built-in keys. If you enable this feature, - * you must implement the function mbedtls_psa_platform_get_builtin_key(). - * See the documentation of that function for more information. - * - * Built-in keys are typically derived from a hardware unique key or - * stored in a secure element. - * - * Requires: MBEDTLS_PSA_CRYPTO_C. - * - * \warning This interface is experimental and may change or be removed - * without notice. - */ -//#define MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS - -/** \def MBEDTLS_PSA_CRYPTO_CLIENT - * - * Enable support for PSA crypto client. - * - * \note This option allows to include the code necessary for a PSA - * crypto client when the PSA crypto implementation is not included in - * the library (MBEDTLS_PSA_CRYPTO_C disabled). The code included is the - * code to set and get PSA key attributes. - * The development of PSA drivers partially relying on the library to - * fulfill the hardware gaps is another possible usage of this option. - * - * \warning This interface is experimental and may change or be removed - * without notice. - */ -//#define MBEDTLS_PSA_CRYPTO_CLIENT - -/** \def MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG - * - * Make the PSA Crypto module use an external random generator provided - * by a driver, instead of Mbed TLS's entropy and DRBG modules. - * - * \note This random generator must deliver random numbers with cryptographic - * quality and high performance. It must supply unpredictable numbers - * with a uniform distribution. The implementation of this function - * is responsible for ensuring that the random generator is seeded - * with sufficient entropy. If you have a hardware TRNG which is slow - * or delivers non-uniform output, declare it as an entropy source - * with mbedtls_entropy_add_source() instead of enabling this option. - * - * If you enable this option, you must configure the type - * ::mbedtls_psa_external_random_context_t in psa/crypto_platform.h - * and define a function called mbedtls_psa_external_get_random() - * with the following prototype: - * ``` - * psa_status_t mbedtls_psa_external_get_random( - * mbedtls_psa_external_random_context_t *context, - * uint8_t *output, size_t output_size, size_t *output_length); - * ); - * ``` - * The \c context value is initialized to 0 before the first call. - * The function must fill the \c output buffer with \c output_size bytes - * of random data and set \c *output_length to \c output_size. - * - * Requires: MBEDTLS_PSA_CRYPTO_C - * - * \warning If you enable this option, code that uses the PSA cryptography - * interface will not use any of the entropy sources set up for - * the entropy module, nor the NV seed that MBEDTLS_ENTROPY_NV_SEED - * enables. - * - * \note This option is experimental and may be removed without notice. - */ -//#define MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG - -/** - * \def MBEDTLS_PSA_CRYPTO_SPM - * - * When MBEDTLS_PSA_CRYPTO_SPM is defined, the code is built for SPM (Secure - * Partition Manager) integration which separates the code into two parts: a - * NSPE (Non-Secure Process Environment) and an SPE (Secure Process - * Environment). - * - * If you enable this option, your build environment must include a header - * file `"crypto_spe.h"` (either in the `psa` subdirectory of the Mbed TLS - * header files, or in another directory on the compiler's include search - * path). Alternatively, your platform may customize the header - * `psa/crypto_platform.h`, in which case it can skip or replace the - * inclusion of `"crypto_spe.h"`. - * - * Module: library/psa_crypto.c - * Requires: MBEDTLS_PSA_CRYPTO_C - * - */ -//#define MBEDTLS_PSA_CRYPTO_SPM - -/** - * \def MBEDTLS_PSA_KEY_STORE_DYNAMIC - * - * Dynamically resize the PSA key store to accommodate any number of - * volatile keys (until the heap memory is exhausted). - * - * If this option is disabled, the key store has a fixed size - * #MBEDTLS_PSA_KEY_SLOT_COUNT for volatile keys and loaded persistent keys - * together. - * - * This option has no effect when #MBEDTLS_PSA_CRYPTO_C is disabled. - * - * Module: library/psa_crypto.c - * Requires: MBEDTLS_PSA_CRYPTO_C - */ -#define MBEDTLS_PSA_KEY_STORE_DYNAMIC - -/** - * Uncomment to enable p256-m. This is an alternative implementation of - * key generation, ECDH and (randomized) ECDSA on the curve SECP256R1. - * Compared to the default implementation: - * - * - p256-m has a much smaller code size and RAM footprint. - * - p256-m is only available via the PSA API. This includes the pk module - * when #MBEDTLS_USE_PSA_CRYPTO is enabled. - * - p256-m does not support deterministic ECDSA, EC-JPAKE, custom protocols - * over the core arithmetic, or deterministic derivation of keys. - * - * We recommend enabling this option if your application uses the PSA API - * and the only elliptic curve support it needs is ECDH and ECDSA over - * SECP256R1. - * - * If you enable this option, you do not need to enable any ECC-related - * MBEDTLS_xxx option. You do need to separately request support for the - * cryptographic mechanisms through the PSA API: - * - #MBEDTLS_PSA_CRYPTO_C and #MBEDTLS_PSA_CRYPTO_CONFIG for PSA-based - * configuration; - * - #MBEDTLS_USE_PSA_CRYPTO if you want to use p256-m from PK, X.509 or TLS; - * - #PSA_WANT_ECC_SECP_R1_256; - * - #PSA_WANT_ALG_ECDH and/or #PSA_WANT_ALG_ECDSA as needed; - * - #PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY, #PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC, - * #PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT, - * #PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT and/or - * #PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE as needed. - * - * \note To benefit from the smaller code size of p256-m, make sure that you - * do not enable any ECC-related option not supported by p256-m: this - * would cause the built-in ECC implementation to be built as well, in - * order to provide the required option. - * Make sure #PSA_WANT_ALG_DETERMINISTIC_ECDSA, #PSA_WANT_ALG_JPAKE and - * #PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE, and curves other than - * SECP256R1 are disabled as they are not supported by this driver. - * Also, avoid defining #MBEDTLS_PK_PARSE_EC_COMPRESSED or - * #MBEDTLS_PK_PARSE_EC_EXTENDED as those currently require a subset of - * the built-in ECC implementation, see docs/driver-only-builds.md. - */ -//#define MBEDTLS_PSA_P256M_DRIVER_ENABLED - -/** - * \def MBEDTLS_PSA_INJECT_ENTROPY - * - * Enable support for entropy injection at first boot. This feature is - * required on systems that do not have a built-in entropy source (TRNG). - * This feature is currently not supported on systems that have a built-in - * entropy source. - * - * Requires: MBEDTLS_PSA_CRYPTO_STORAGE_C, MBEDTLS_ENTROPY_NV_SEED - * - */ -//#define MBEDTLS_PSA_INJECT_ENTROPY - -/** - * \def MBEDTLS_PSA_ASSUME_EXCLUSIVE_BUFFERS - * - * Assume all buffers passed to PSA functions are owned exclusively by the - * PSA function and are not stored in shared memory. - * - * This option may be enabled if all buffers passed to any PSA function reside - * in memory that is accessible only to the PSA function during its execution. - * - * This option MUST be disabled whenever buffer arguments are in memory shared - * with an untrusted party, for example where arguments to PSA calls are passed - * across a trust boundary. - * - * \note Enabling this option reduces memory usage and code size. - * - * \note Enabling this option causes overlap of input and output buffers - * not to be supported by PSA functions. - */ -//#define MBEDTLS_PSA_ASSUME_EXCLUSIVE_BUFFERS - -/** - * \def MBEDTLS_RSA_NO_CRT - * - * Do not use the Chinese Remainder Theorem - * for the RSA private operation. - * - * Uncomment this macro to disable the use of CRT in RSA. - * - */ -//#define MBEDTLS_RSA_NO_CRT - -/** - * \def MBEDTLS_SELF_TEST - * - * Enable the checkup functions (*_self_test). - */ -#define MBEDTLS_SELF_TEST - -/** - * \def MBEDTLS_SHA256_SMALLER - * - * Enable an implementation of SHA-256 that has lower ROM footprint but also - * lower performance. - * - * The default implementation is meant to be a reasonable compromise between - * performance and size. This version optimizes more aggressively for size at - * the expense of performance. Eg on Cortex-M4 it reduces the size of - * mbedtls_sha256_process() from ~2KB to ~0.5KB for a performance hit of about - * 30%. - * - * Uncomment to enable the smaller implementation of SHA256. - */ -//#define MBEDTLS_SHA256_SMALLER - -/** - * \def MBEDTLS_SHA512_SMALLER - * - * Enable an implementation of SHA-512 that has lower ROM footprint but also - * lower performance. - * - * Uncomment to enable the smaller implementation of SHA512. - */ -//#define MBEDTLS_SHA512_SMALLER - -/** - * \def MBEDTLS_SSL_ALL_ALERT_MESSAGES - * - * Enable sending of alert messages in case of encountered errors as per RFC. - * If you choose not to send the alert messages, Mbed TLS can still communicate - * with other servers, only debugging of failures is harder. - * - * The advantage of not sending alert messages, is that no information is given - * about reasons for failures thus preventing adversaries of gaining intel. - * - * Enable sending of all alert messages - */ -#define MBEDTLS_SSL_ALL_ALERT_MESSAGES - -/** - * \def MBEDTLS_SSL_DTLS_CONNECTION_ID - * - * Enable support for the DTLS Connection ID (CID) extension, - * which allows to identify DTLS connections across changes - * in the underlying transport. The CID functionality is described - * in RFC 9146. - * - * Setting this option enables the SSL APIs `mbedtls_ssl_set_cid()`, - * mbedtls_ssl_get_own_cid()`, `mbedtls_ssl_get_peer_cid()` and - * `mbedtls_ssl_conf_cid()`. See the corresponding documentation for - * more information. - * - * The maximum lengths of outgoing and incoming CIDs can be configured - * through the options - * - MBEDTLS_SSL_CID_OUT_LEN_MAX - * - MBEDTLS_SSL_CID_IN_LEN_MAX. - * - * Requires: MBEDTLS_SSL_PROTO_DTLS - * - * Uncomment to enable the Connection ID extension. - */ -#define MBEDTLS_SSL_DTLS_CONNECTION_ID - - -/** - * \def MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT - * - * Defines whether RFC 9146 (default) or the legacy version - * (version draft-ietf-tls-dtls-connection-id-05, - * https://tools.ietf.org/html/draft-ietf-tls-dtls-connection-id-05) - * is used. - * - * Set the value to 0 for the standard version, and - * 1 for the legacy draft version. - * - * \deprecated Support for the legacy version of the DTLS - * Connection ID feature is deprecated. Please - * switch to the standardized version defined - * in RFC 9146 enabled by utilizing - * MBEDTLS_SSL_DTLS_CONNECTION_ID without use - * of MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT. - * - * Requires: MBEDTLS_SSL_DTLS_CONNECTION_ID - */ -#define MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT 0 - -/** - * \def MBEDTLS_SSL_ASYNC_PRIVATE - * - * Enable asynchronous external private key operations in SSL. This allows - * you to configure an SSL connection to call an external cryptographic - * module to perform private key operations instead of performing the - * operation inside the library. - * - * Requires: MBEDTLS_X509_CRT_PARSE_C - */ -//#define MBEDTLS_SSL_ASYNC_PRIVATE - -/** \def MBEDTLS_SSL_CLI_ALLOW_WEAK_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * - * In TLS clients, when a client authenticates a server through its - * certificate, the client normally checks three things: - * - the certificate chain must be valid; - * - the chain must start from a trusted CA; - * - the certificate must cover the server name that is expected by the client. - * - * Omitting any of these checks is generally insecure, and can allow a - * malicious server to impersonate a legitimate server. - * - * The third check may be safely skipped in some unusual scenarios, - * such as networks where eavesdropping is a risk but not active attacks, - * or a private PKI where the client equally trusts all servers that are - * accredited by the root CA. - * - * You should call mbedtls_ssl_set_hostname() with the expected server name - * before starting a TLS handshake on a client (unless the client is - * set up to only use PSK-based authentication, which does not rely on the - * host name). This configuration option controls what happens if a TLS client - * is configured with the authentication mode #MBEDTLS_SSL_VERIFY_REQUIRED - * (default), certificate authentication is enabled and the client does not - * call mbedtls_ssl_set_hostname(): - * - * - If this option is unset (default), the connection attempt is aborted - * with the error #MBEDTLS_ERR_SSL_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME. - * - If this option is set, the TLS library does not check the server name - * that the certificate is valid for. This is the historical behavior - * of Mbed TLS, but may be insecure as explained above. - * - * Enable this option for strict backward compatibility if you have - * determined that it is secure in the scenario where you are using - * Mbed TLS. - * - * \deprecated This option exists only for backward compatibility and will - * be removed in the next major version of Mbed TLS. - * - */ -//#define MBEDTLS_SSL_CLI_ALLOW_WEAK_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - -/** - * \def MBEDTLS_SSL_CONTEXT_SERIALIZATION - * - * Enable serialization of the TLS context structures, through use of the - * functions mbedtls_ssl_context_save() and mbedtls_ssl_context_load(). - * - * This pair of functions allows one side of a connection to serialize the - * context associated with the connection, then free or re-use that context - * while the serialized state is persisted elsewhere, and finally deserialize - * that state to a live context for resuming read/write operations on the - * connection. From a protocol perspective, the state of the connection is - * unaffected, in particular this is entirely transparent to the peer. - * - * Note: this is distinct from TLS session resumption, which is part of the - * protocol and fully visible by the peer. TLS session resumption enables - * establishing new connections associated to a saved session with shorter, - * lighter handshakes, while context serialization is a local optimization in - * handling a single, potentially long-lived connection. - * - * Enabling these APIs makes some SSL structures larger, as 64 extra bytes are - * saved after the handshake to allow for more efficient serialization, so if - * you don't need this feature you'll save RAM by disabling it. - * - * Requires: MBEDTLS_GCM_C or MBEDTLS_CCM_C or MBEDTLS_CHACHAPOLY_C - * - * Comment to disable the context serialization APIs. - */ -#define MBEDTLS_SSL_CONTEXT_SERIALIZATION - -/** - * \def MBEDTLS_SSL_DEBUG_ALL - * - * Enable the debug messages in SSL module for all issues. - * Debug messages have been disabled in some places to prevent timing - * attacks due to (unbalanced) debugging function calls. - * - * If you need all error reporting you should enable this during debugging, - * but remove this for production servers that should log as well. - * - * Uncomment this macro to report all debug messages on errors introducing - * a timing side-channel. - * - */ -//#define MBEDTLS_SSL_DEBUG_ALL - -/** \def MBEDTLS_SSL_ENCRYPT_THEN_MAC - * - * Enable support for Encrypt-then-MAC, RFC 7366. - * - * This allows peers that both support it to use a more robust protection for - * ciphersuites using CBC, providing deep resistance against timing attacks - * on the padding or underlying cipher. - * - * This only affects CBC ciphersuites, and is useless if none is defined. - * - * Requires: MBEDTLS_SSL_PROTO_TLS1_2 - * - * Comment this macro to disable support for Encrypt-then-MAC - */ -#define MBEDTLS_SSL_ENCRYPT_THEN_MAC - -/** \def MBEDTLS_SSL_EXTENDED_MASTER_SECRET - * - * Enable support for RFC 7627: Session Hash and Extended Master Secret - * Extension. - * - * This was introduced as "the proper fix" to the Triple Handshake family of - * attacks, but it is recommended to always use it (even if you disable - * renegotiation), since it actually fixes a more fundamental issue in the - * original SSL/TLS design, and has implications beyond Triple Handshake. - * - * Requires: MBEDTLS_SSL_PROTO_TLS1_2 - * - * Comment this macro to disable support for Extended Master Secret. - */ -#define MBEDTLS_SSL_EXTENDED_MASTER_SECRET - -/** - * \def MBEDTLS_SSL_KEEP_PEER_CERTIFICATE - * - * This option controls the availability of the API mbedtls_ssl_get_peer_cert() - * giving access to the peer's certificate after completion of the handshake. - * - * Unless you need mbedtls_ssl_peer_cert() in your application, it is - * recommended to disable this option for reduced RAM usage. - * - * \note If this option is disabled, mbedtls_ssl_get_peer_cert() is still - * defined, but always returns \c NULL. - * - * \note This option has no influence on the protection against the - * triple handshake attack. Even if it is disabled, Mbed TLS will - * still ensure that certificates do not change during renegotiation, - * for example by keeping a hash of the peer's certificate. - * - * \note This option is required if MBEDTLS_SSL_PROTO_TLS1_3 is set. - * - * Comment this macro to disable storing the peer's certificate - * after the handshake. - */ -#define MBEDTLS_SSL_KEEP_PEER_CERTIFICATE - -/** - * \def MBEDTLS_SSL_KEYING_MATERIAL_EXPORT - * - * When this option is enabled, the client and server can extract additional - * shared symmetric keys after an SSL handshake using the function - * mbedtls_ssl_export_keying_material(). - * - * The process for deriving the keys is specified in RFC 5705 for TLS 1.2 and - * in RFC 8446, Section 7.5, for TLS 1.3. - * - * Comment this macro to disable mbedtls_ssl_export_keying_material(). - */ -#define MBEDTLS_SSL_KEYING_MATERIAL_EXPORT - -/** - * \def MBEDTLS_SSL_RENEGOTIATION - * - * Enable support for TLS renegotiation. - * - * The two main uses of renegotiation are (1) refresh keys on long-lived - * connections and (2) client authentication after the initial handshake. - * If you don't need renegotiation, it's probably better to disable it, since - * it has been associated with security issues in the past and is easy to - * misuse/misunderstand. - * - * Requires: MBEDTLS_SSL_PROTO_TLS1_2 - * - * Comment this to disable support for renegotiation. - * - * \note Even if this option is disabled, both client and server are aware - * of the Renegotiation Indication Extension (RFC 5746) used to - * prevent the SSL renegotiation attack (see RFC 5746 Sect. 1). - * (See \c mbedtls_ssl_conf_legacy_renegotiation for the - * configuration of this extension). - * - */ -#define MBEDTLS_SSL_RENEGOTIATION - -/** - * \def MBEDTLS_SSL_MAX_FRAGMENT_LENGTH - * - * Enable support for RFC 6066 max_fragment_length extension in SSL. - * - * Comment this macro to disable support for the max_fragment_length extension - */ -#define MBEDTLS_SSL_MAX_FRAGMENT_LENGTH - -/** - * \def MBEDTLS_SSL_RECORD_SIZE_LIMIT - * - * Enable support for RFC 8449 record_size_limit extension in SSL (TLS 1.3 only). - * - * Requires: MBEDTLS_SSL_PROTO_TLS1_3 - * - * Uncomment this macro to enable support for the record_size_limit extension - */ -//#define MBEDTLS_SSL_RECORD_SIZE_LIMIT - -/** - * \def MBEDTLS_SSL_PROTO_TLS1_2 - * - * Enable support for TLS 1.2 (and DTLS 1.2 if DTLS is enabled). - * - * Requires: Without MBEDTLS_USE_PSA_CRYPTO: MBEDTLS_MD_C and - * (MBEDTLS_SHA256_C or MBEDTLS_SHA384_C or - * SHA-256 or SHA-512 provided by a PSA driver) - * With MBEDTLS_USE_PSA_CRYPTO: - * PSA_WANT_ALG_SHA_256 or PSA_WANT_ALG_SHA_384 - * - * \warning If building with MBEDTLS_USE_PSA_CRYPTO, or if the hash(es) used - * are only provided by PSA drivers, you must call psa_crypto_init() before - * doing any TLS operations. - * - * Comment this macro to disable support for TLS 1.2 / DTLS 1.2 - */ -#define MBEDTLS_SSL_PROTO_TLS1_2 - -/** - * \def MBEDTLS_SSL_PROTO_TLS1_3 - * - * Enable support for TLS 1.3. - * - * \note See docs/architecture/tls13-support.md for a description of the TLS - * 1.3 support that this option enables. - * - * Requires: MBEDTLS_SSL_KEEP_PEER_CERTIFICATE - * Requires: MBEDTLS_PSA_CRYPTO_C - * - * \note TLS 1.3 uses PSA crypto for cryptographic operations that are - * directly performed by TLS 1.3 code. As a consequence, when TLS 1.3 - * is enabled, a TLS handshake may call psa_crypto_init(), even - * if it ends up negotiating a different TLS version. - * - * \note Cryptographic operations performed indirectly via another module - * (X.509, PK) or by code shared with TLS 1.2 (record protection, - * running handshake hash) only use PSA crypto if - * #MBEDTLS_USE_PSA_CRYPTO is enabled. - * - * \note In multithreaded applications, you must also enable - * #MBEDTLS_THREADING_C, even if individual TLS contexts are not - * shared between threads, unless only one thread ever calls - * TLS functions. - * - * Uncomment this macro to enable the support for TLS 1.3. - */ -#define MBEDTLS_SSL_PROTO_TLS1_3 - -/** - * \def MBEDTLS_SSL_TLS1_3_COMPATIBILITY_MODE - * - * Enable TLS 1.3 middlebox compatibility mode. - * - * As specified in Section D.4 of RFC 8446, TLS 1.3 offers a compatibility - * mode to make a TLS 1.3 connection more likely to pass through middle boxes - * expecting TLS 1.2 traffic. - * - * Turning on the compatibility mode comes at the cost of a few added bytes - * on the wire, but it doesn't affect compatibility with TLS 1.3 implementations - * that don't use it. Therefore, unless transmission bandwidth is critical and - * you know that middlebox compatibility issues won't occur, it is therefore - * recommended to set this option. - * - * Comment to disable compatibility mode for TLS 1.3. If - * MBEDTLS_SSL_PROTO_TLS1_3 is not enabled, this option does not have any - * effect on the build. - * - */ -#define MBEDTLS_SSL_TLS1_3_COMPATIBILITY_MODE - -/** - * \def MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED - * - * Enable TLS 1.3 PSK key exchange mode. - * - * Comment to disable support for the PSK key exchange mode in TLS 1.3. If - * MBEDTLS_SSL_PROTO_TLS1_3 is not enabled, this option does not have any - * effect on the build. - * - */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED - -/** - * \def MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED - * - * Enable TLS 1.3 ephemeral key exchange mode. - * - * Requires: PSA_WANT_ALG_ECDH or PSA_WANT_ALG_FFDH - * MBEDTLS_X509_CRT_PARSE_C - * and at least one of: - * MBEDTLS_ECDSA_C or (MBEDTLS_USE_PSA_CRYPTO and PSA_WANT_ALG_ECDSA) - * MBEDTLS_PKCS1_V21 - * - * Comment to disable support for the ephemeral key exchange mode in TLS 1.3. - * If MBEDTLS_SSL_PROTO_TLS1_3 is not enabled, this option does not have any - * effect on the build. - * - */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED - -/** - * \def MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED - * - * Enable TLS 1.3 PSK ephemeral key exchange mode. - * - * Requires: PSA_WANT_ALG_ECDH or PSA_WANT_ALG_FFDH - * - * Comment to disable support for the PSK ephemeral key exchange mode in - * TLS 1.3. If MBEDTLS_SSL_PROTO_TLS1_3 is not enabled, this option does not - * have any effect on the build. - * - */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED - -/** - * \def MBEDTLS_SSL_EARLY_DATA - * - * Enable support for RFC 8446 TLS 1.3 early data. - * - * Requires: MBEDTLS_SSL_SESSION_TICKETS and either - * MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED or - * MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED - * - * Comment this to disable support for early data. If MBEDTLS_SSL_PROTO_TLS1_3 - * is not enabled, this option does not have any effect on the build. - * - * \note The maximum amount of early data can be set with - * MBEDTLS_SSL_MAX_EARLY_DATA_SIZE. - * - */ -//#define MBEDTLS_SSL_EARLY_DATA - -/** - * \def MBEDTLS_SSL_PROTO_DTLS - * - * Enable support for DTLS (all available versions). - * - * Enable this and MBEDTLS_SSL_PROTO_TLS1_2 to enable DTLS 1.2. - * - * Requires: MBEDTLS_SSL_PROTO_TLS1_2 - * - * Comment this macro to disable support for DTLS - */ -#define MBEDTLS_SSL_PROTO_DTLS - -/** - * \def MBEDTLS_SSL_ALPN - * - * Enable support for RFC 7301 Application Layer Protocol Negotiation. - * - * Comment this macro to disable support for ALPN. - */ -#define MBEDTLS_SSL_ALPN - -/** - * \def MBEDTLS_SSL_DTLS_ANTI_REPLAY - * - * Enable support for the anti-replay mechanism in DTLS. - * - * Requires: MBEDTLS_SSL_TLS_C - * MBEDTLS_SSL_PROTO_DTLS - * - * \warning Disabling this is often a security risk! - * See mbedtls_ssl_conf_dtls_anti_replay() for details. - * - * Comment this to disable anti-replay in DTLS. - */ -#define MBEDTLS_SSL_DTLS_ANTI_REPLAY - -/** - * \def MBEDTLS_SSL_DTLS_HELLO_VERIFY - * - * Enable support for HelloVerifyRequest on DTLS servers. - * - * This feature is highly recommended to prevent DTLS servers being used as - * amplifiers in DoS attacks against other hosts. It should always be enabled - * unless you know for sure amplification cannot be a problem in the - * environment in which your server operates. - * - * \warning Disabling this can be a security risk! (see above) - * - * Requires: MBEDTLS_SSL_PROTO_DTLS - * - * Comment this to disable support for HelloVerifyRequest. - */ -#define MBEDTLS_SSL_DTLS_HELLO_VERIFY - -/** - * \def MBEDTLS_SSL_DTLS_SRTP - * - * Enable support for negotiation of DTLS-SRTP (RFC 5764) - * through the use_srtp extension. - * - * \note This feature provides the minimum functionality required - * to negotiate the use of DTLS-SRTP and to allow the derivation of - * the associated SRTP packet protection key material. - * In particular, the SRTP packet protection itself, as well as the - * demultiplexing of RTP and DTLS packets at the datagram layer - * (see Section 5 of RFC 5764), are not handled by this feature. - * Instead, after successful completion of a handshake negotiating - * the use of DTLS-SRTP, the extended key exporter API - * mbedtls_ssl_conf_export_keys_cb() should be used to implement - * the key exporter described in Section 4.2 of RFC 5764 and RFC 5705 - * (this is implemented in the SSL example programs). - * The resulting key should then be passed to an SRTP stack. - * - * Setting this option enables the runtime API - * mbedtls_ssl_conf_dtls_srtp_protection_profiles() - * through which the supported DTLS-SRTP protection - * profiles can be configured. You must call this API at - * runtime if you wish to negotiate the use of DTLS-SRTP. - * - * Requires: MBEDTLS_SSL_PROTO_DTLS - * - * Uncomment this to enable support for use_srtp extension. - */ -//#define MBEDTLS_SSL_DTLS_SRTP - -/** - * \def MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE - * - * Enable server-side support for clients that reconnect from the same port. - * - * Some clients unexpectedly close the connection and try to reconnect using the - * same source port. This needs special support from the server to handle the - * new connection securely, as described in section 4.2.8 of RFC 6347. This - * flag enables that support. - * - * Requires: MBEDTLS_SSL_DTLS_HELLO_VERIFY - * - * Comment this to disable support for clients reusing the source port. - */ -#define MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE - -/** - * \def MBEDTLS_SSL_SESSION_TICKETS - * - * Enable support for RFC 5077 session tickets in SSL. - * Client-side, provides full support for session tickets (maintenance of a - * session store remains the responsibility of the application, though). - * Server-side, you also need to provide callbacks for writing and parsing - * tickets, including authenticated encryption and key management. Example - * callbacks are provided by MBEDTLS_SSL_TICKET_C. - * - * Comment this macro to disable support for SSL session tickets - */ -#define MBEDTLS_SSL_SESSION_TICKETS - -/** - * \def MBEDTLS_SSL_SERVER_NAME_INDICATION - * - * Enable support for RFC 6066 server name indication (SNI) in SSL. - * - * Requires: MBEDTLS_X509_CRT_PARSE_C - * - * Comment this macro to disable support for server name indication in SSL - */ -#define MBEDTLS_SSL_SERVER_NAME_INDICATION - -/** - * \def MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH - * - * When this option is enabled, the SSL buffer will be resized automatically - * based on the negotiated maximum fragment length in each direction. - * - * Requires: MBEDTLS_SSL_MAX_FRAGMENT_LENGTH - */ -//#define MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH - -/** - * \def MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - * - * Enable testing of the constant-flow nature of some sensitive functions with - * clang's MemorySanitizer. This causes some existing tests to also test - * this non-functional property of the code under test. - * - * This setting requires compiling with clang -fsanitize=memory. The test - * suites can then be run normally. - * - * \warning This macro is only used for extended testing; it is not considered - * part of the library's API, so it may change or disappear at any time. - * - * Uncomment to enable testing of the constant-flow nature of selected code. - */ -//#define MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN - -/** - * \def MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - * - * Enable testing of the constant-flow nature of some sensitive functions with - * valgrind's memcheck tool. This causes some existing tests to also test - * this non-functional property of the code under test. - * - * This setting requires valgrind headers for building, and is only useful for - * testing if the tests suites are run with valgrind's memcheck. This can be - * done for an individual test suite with 'valgrind ./test_suite_xxx', or when - * using CMake, this can be done for all test suites with 'make memcheck'. - * - * \warning This macro is only used for extended testing; it is not considered - * part of the library's API, so it may change or disappear at any time. - * - * Uncomment to enable testing of the constant-flow nature of selected code. - */ -//#define MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND - -/** - * \def MBEDTLS_TEST_HOOKS - * - * Enable features for invasive testing such as introspection functions and - * hooks for fault injection. This enables additional unit tests. - * - * Merely enabling this feature should not change the behavior of the product. - * It only adds new code, and new branching points where the default behavior - * is the same as when this feature is disabled. - * However, this feature increases the attack surface: there is an added - * risk of vulnerabilities, and more gadgets that can make exploits easier. - * Therefore this feature must never be enabled in production. - * - * See `docs/architecture/testing/mbed-crypto-invasive-testing.md` for more - * information. - * - * Uncomment to enable invasive tests. - */ -//#define MBEDTLS_TEST_HOOKS - -/** - * \def MBEDTLS_THREADING_ALT - * - * Provide your own alternate implementation of threading primitives - * for mutexes. If you enable this option: - * - * - Provide a header file `"threading_alt.h"`, defining the - * type `mbedtls_threading_mutex_t` of mutex objects. - * - * - Call the function mbedtls_threading_set_alt() in your application - * before calling any other library function (in particular before - * calling psa_crypto_init(), performing an asymmetric cryptography - * operation, or starting a TLS connection). - * - * See mbedtls/threading.h for more details, especially the documentation - * of mbedtls_threading_set_alt(). - * - * Requires: MBEDTLS_THREADING_C - * - * Uncomment this to allow your own alternate threading implementation. - */ -//#define MBEDTLS_THREADING_ALT - -/** - * \def MBEDTLS_THREADING_PTHREAD - * - * Enable the pthread wrapper layer for the threading layer. - * - * Requires: MBEDTLS_THREADING_C - * - * Uncomment this to enable pthread mutexes. - */ -//#define MBEDTLS_THREADING_PTHREAD - -/** - * \def MBEDTLS_USE_PSA_CRYPTO - * - * Make the X.509 and TLS libraries use PSA for cryptographic operations as - * much as possible, and enable new APIs for using keys handled by PSA Crypto. - * - * \note Development of this option is currently in progress, and parts of Mbed - * TLS's X.509 and TLS modules are not ported to PSA yet. However, these parts - * will still continue to work as usual, so enabling this option should not - * break backwards compatibility. - * - * \warning If you enable this option, you need to call `psa_crypto_init()` - * before calling any function from the SSL/TLS, X.509 or PK modules, except - * for the various mbedtls_xxx_init() functions which can be called at any time. - * - * \warning In multithreaded applications, you must also enable - * #MBEDTLS_THREADING_C, unless only one thread ever calls PSA functions - * (`psa_xxx()`), including indirect calls through SSL/TLS, X.509 or PK. - * - * \note An important and desirable effect of this option is that it allows - * PK, X.509 and TLS to take advantage of PSA drivers. For example, enabling - * this option is what allows use of drivers for ECDSA, ECDH and EC J-PAKE in - * those modules. However, note that even with this option disabled, some code - * in PK, X.509, TLS or the crypto library might still use PSA drivers, if it - * can determine it's safe to do so; currently that's the case for hashes. - * - * \note See docs/use-psa-crypto.md for a complete description this option. - * - * Requires: MBEDTLS_PSA_CRYPTO_C. - * - * Uncomment this to enable internal use of PSA Crypto and new associated APIs. - */ -//#define MBEDTLS_USE_PSA_CRYPTO - -/** - * \def MBEDTLS_PSA_CRYPTO_CONFIG - * - * This setting allows support for cryptographic mechanisms through the PSA - * API to be configured separately from support through the mbedtls API. - * - * When this option is disabled, the PSA API exposes the cryptographic - * mechanisms that can be implemented on top of the `mbedtls_xxx` API - * configured with `MBEDTLS_XXX` symbols. - * - * When this option is enabled, the PSA API exposes the cryptographic - * mechanisms requested by the `PSA_WANT_XXX` symbols defined in - * include/psa/crypto_config.h. The corresponding `MBEDTLS_XXX` settings are - * automatically enabled if required (i.e. if no PSA driver provides the - * mechanism). You may still freely enable additional `MBEDTLS_XXX` symbols - * in mbedtls_config.h. - * - * If the symbol #MBEDTLS_PSA_CRYPTO_CONFIG_FILE is defined, it specifies - * an alternative header to include instead of include/psa/crypto_config.h. - * - * \warning This option is experimental, in that the set of `PSA_WANT_XXX` - * symbols is not completely finalized yet, and the configuration - * tooling is not ideally adapted to having two separate configuration - * files. - * Future minor releases of Mbed TLS may make minor changes to those - * symbols, but we will endeavor to provide a transition path. - * Nonetheless, this option is considered mature enough to use in - * production, as long as you accept that you may need to make - * minor changes to psa/crypto_config.h when upgrading Mbed TLS. - */ -//#define MBEDTLS_PSA_CRYPTO_CONFIG - -/** - * \def MBEDTLS_VERSION_FEATURES - * - * Allow run-time checking of compile-time enabled features. Thus allowing users - * to check at run-time if the library is for instance compiled with threading - * support via mbedtls_version_check_feature(). - * - * Requires: MBEDTLS_VERSION_C - * - * Comment this to disable run-time checking and save ROM space - */ -#define MBEDTLS_VERSION_FEATURES - -/** - * \def MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK - * - * If set, this enables the X.509 API `mbedtls_x509_crt_verify_with_ca_cb()` - * and the SSL API `mbedtls_ssl_conf_ca_cb()` which allow users to configure - * the set of trusted certificates through a callback instead of a linked - * list. - * - * This is useful for example in environments where a large number of trusted - * certificates is present and storing them in a linked list isn't efficient - * enough, or when the set of trusted certificates changes frequently. - * - * See the documentation of `mbedtls_x509_crt_verify_with_ca_cb()` and - * `mbedtls_ssl_conf_ca_cb()` for more information. - * - * Requires: MBEDTLS_X509_CRT_PARSE_C - * - * Uncomment to enable trusted certificate callbacks. - */ -//#define MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK - -/** - * \def MBEDTLS_X509_REMOVE_INFO - * - * Disable mbedtls_x509_*_info() and related APIs. - * - * Uncomment to omit mbedtls_x509_*_info(), as well as mbedtls_debug_print_crt() - * and other functions/constants only used by these functions, thus reducing - * the code footprint by several KB. - */ -//#define MBEDTLS_X509_REMOVE_INFO - -/** - * \def MBEDTLS_X509_RSASSA_PSS_SUPPORT - * - * Enable parsing and verification of X.509 certificates, CRLs and CSRS - * signed with RSASSA-PSS (aka PKCS#1 v2.1). - * - * Requires: MBEDTLS_PKCS1_V21 - * - * Comment this macro to disallow using RSASSA-PSS in certificates. - */ -#define MBEDTLS_X509_RSASSA_PSS_SUPPORT -/** \} name SECTION: Mbed TLS feature support */ - -/** - * \name SECTION: Mbed TLS modules - * - * This section enables or disables entire modules in Mbed TLS - * \{ - */ - -/** - * \def MBEDTLS_AESNI_C - * - * Enable AES-NI support on x86-64 or x86-32. - * - * \note AESNI is only supported with certain compilers and target options: - * - Visual Studio: supported - * - GCC, x86-64, target not explicitly supporting AESNI: - * requires MBEDTLS_HAVE_ASM. - * - GCC, x86-32, target not explicitly supporting AESNI: - * not supported. - * - GCC, x86-64 or x86-32, target supporting AESNI: supported. - * For this assembly-less implementation, you must currently compile - * `library/aesni.c` and `library/aes.c` with machine options to enable - * SSE2 and AESNI instructions: `gcc -msse2 -maes -mpclmul` or - * `clang -maes -mpclmul`. - * - Non-x86 targets: this option is silently ignored. - * - Other compilers: this option is silently ignored. - * - * \note - * Above, "GCC" includes compatible compilers such as Clang. - * The limitations on target support are likely to be relaxed in the future. - * - * Module: library/aesni.c - * Caller: library/aes.c - * - * Requires: MBEDTLS_HAVE_ASM (on some platforms, see note) - * - * This modules adds support for the AES-NI instructions on x86. - */ -#define MBEDTLS_AESNI_C - -/** - * \def MBEDTLS_AESCE_C - * - * Enable AES cryptographic extension support on Armv8. - * - * Module: library/aesce.c - * Caller: library/aes.c - * - * Requires: MBEDTLS_AES_C - * - * \warning Runtime detection only works on Linux. For non-Linux operating - * system, Armv8-A Cryptographic Extensions must be supported by - * the CPU when this option is enabled. - * - * \note Minimum compiler versions for this feature when targeting aarch64 - * are Clang 4.0; armclang 6.6; GCC 6.0; or MSVC 2019 version 16.11.2. - * Minimum compiler versions for this feature when targeting 32-bit - * Arm or Thumb are Clang 11.0; armclang 6.20; or GCC 6.0. - * - * \note \c CFLAGS must be set to a minimum of \c -march=armv8-a+crypto for - * armclang <= 6.9 - * - * This module adds support for the AES Armv8-A Cryptographic Extensions on Armv8 systems. - */ -#define MBEDTLS_AESCE_C - -/** - * \def MBEDTLS_AES_C - * - * Enable the AES block cipher. - * - * Module: library/aes.c - * Caller: library/cipher.c - * library/pem.c - * library/ctr_drbg.c - * - * This module enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA - * MBEDTLS_TLS_PSK_WITH_AES_256_GCM_SHA384 - * MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA384 - * MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA - * MBEDTLS_TLS_PSK_WITH_AES_128_GCM_SHA256 - * MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA256 - * MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA - * - * PEM_PARSE uses AES for decrypting encrypted keys. - */ -#define MBEDTLS_AES_C - -/** - * \def MBEDTLS_ASN1_PARSE_C - * - * Enable the generic ASN1 parser. - * - * Module: library/asn1.c - * Caller: library/x509.c - * library/dhm.c - * library/pkcs12.c - * library/pkcs5.c - * library/pkparse.c - */ -#define MBEDTLS_ASN1_PARSE_C - -/** - * \def MBEDTLS_ASN1_WRITE_C - * - * Enable the generic ASN1 writer. - * - * Module: library/asn1write.c - * Caller: library/ecdsa.c - * library/pkwrite.c - * library/x509_create.c - * library/x509write_crt.c - * library/x509write_csr.c - */ -#define MBEDTLS_ASN1_WRITE_C - -/** - * \def MBEDTLS_BASE64_C - * - * Enable the Base64 module. - * - * Module: library/base64.c - * Caller: library/pem.c - * - * This module is required for PEM support (required by X.509). - */ -#define MBEDTLS_BASE64_C - -/** - * \def MBEDTLS_BLOCK_CIPHER_NO_DECRYPT - * - * Remove decryption operation for AES, ARIA and Camellia block cipher. - * - * \note This feature is incompatible with insecure block cipher, - * MBEDTLS_DES_C, and cipher modes which always require decryption - * operation, MBEDTLS_CIPHER_MODE_CBC, MBEDTLS_CIPHER_MODE_XTS and - * MBEDTLS_NIST_KW_C. When #MBEDTLS_PSA_CRYPTO_CONFIG is enabled, - * this feature is incompatible with following supported PSA equivalence, - * PSA_WANT_ALG_ECB_NO_PADDING, PSA_WANT_ALG_CBC_NO_PADDING, - * PSA_WANT_ALG_CBC_PKCS7 and PSA_WANT_KEY_TYPE_DES. - * - * Module: library/aes.c - * library/aesce.c - * library/aesni.c - * library/aria.c - * library/camellia.c - * library/cipher.c - */ -//#define MBEDTLS_BLOCK_CIPHER_NO_DECRYPT - -/** - * \def MBEDTLS_BIGNUM_C - * - * Enable the multi-precision integer library. - * - * Module: library/bignum.c - * library/bignum_core.c - * library/bignum_mod.c - * library/bignum_mod_raw.c - * Caller: library/dhm.c - * library/ecp.c - * library/ecdsa.c - * library/rsa.c - * library/rsa_alt_helpers.c - * library/ssl_tls.c - * - * This module is required for RSA, DHM and ECC (ECDH, ECDSA) support. - */ -#define MBEDTLS_BIGNUM_C - -/** - * \def MBEDTLS_CAMELLIA_C - * - * Enable the Camellia block cipher. - * - * Module: library/camellia.c - * Caller: library/cipher.c - * - * This module enables the following ciphersuites (if other requisites are - * enabled as well): - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 - * MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 - */ -#define MBEDTLS_CAMELLIA_C - -/** - * \def MBEDTLS_ARIA_C - * - * Enable the ARIA block cipher. - * - * Module: library/aria.c - * Caller: library/cipher.c - * - * This module enables the following ciphersuites (if other requisites are - * enabled as well): - * - * MBEDTLS_TLS_RSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_PSK_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_PSK_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 - * MBEDTLS_TLS_PSK_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_PSK_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 - * MBEDTLS_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 - * MBEDTLS_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 - * MBEDTLS_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 - */ -#define MBEDTLS_ARIA_C - -/** - * \def MBEDTLS_CCM_C - * - * Enable the Counter with CBC-MAC (CCM) mode for 128-bit block cipher. - * - * Module: library/ccm.c - * - * Requires: MBEDTLS_CIPHER_C, MBEDTLS_AES_C or MBEDTLS_CAMELLIA_C or - * MBEDTLS_ARIA_C - * - * This module enables the AES-CCM ciphersuites, if other requisites are - * enabled as well. - */ -#define MBEDTLS_CCM_C - -/** - * \def MBEDTLS_CHACHA20_C - * - * Enable the ChaCha20 stream cipher. - * - * Module: library/chacha20.c - */ -#define MBEDTLS_CHACHA20_C - -/** - * \def MBEDTLS_CHACHAPOLY_C - * - * Enable the ChaCha20-Poly1305 AEAD algorithm. - * - * Module: library/chachapoly.c - * - * This module requires: MBEDTLS_CHACHA20_C, MBEDTLS_POLY1305_C - */ -#define MBEDTLS_CHACHAPOLY_C - -/** - * \def MBEDTLS_CIPHER_C - * - * Enable the generic cipher layer. - * - * Module: library/cipher.c - * Caller: library/ccm.c - * library/cmac.c - * library/gcm.c - * library/nist_kw.c - * library/pkcs12.c - * library/pkcs5.c - * library/psa_crypto_aead.c - * library/psa_crypto_mac.c - * library/ssl_ciphersuites.c - * library/ssl_msg.c - * library/ssl_ticket.c (unless MBEDTLS_USE_PSA_CRYPTO is enabled) - * Auto-enabled by: MBEDTLS_PSA_CRYPTO_C depending on which ciphers are enabled - * (see the documentation of that option for details). - * - * Uncomment to enable generic cipher wrappers. - */ -#define MBEDTLS_CIPHER_C - -/** - * \def MBEDTLS_CMAC_C - * - * Enable the CMAC (Cipher-based Message Authentication Code) mode for block - * ciphers. - * - * \note When #MBEDTLS_CMAC_ALT is active, meaning that the underlying - * implementation of the CMAC algorithm is provided by an alternate - * implementation, that alternate implementation may opt to not support - * AES-192 or 3DES as underlying block ciphers for the CMAC operation. - * - * Module: library/cmac.c - * - * Requires: MBEDTLS_CIPHER_C, MBEDTLS_AES_C or MBEDTLS_DES_C - * - */ -#define MBEDTLS_CMAC_C - -/** - * \def MBEDTLS_CTR_DRBG_C - * - * Enable the CTR_DRBG AES-based random generator. - * The CTR_DRBG generator uses AES-256 by default. - * To use AES-128 instead, enable \c MBEDTLS_CTR_DRBG_USE_128_BIT_KEY above. - * - * AES support can either be achieved through builtin (MBEDTLS_AES_C) or PSA. - * Builtin is the default option when MBEDTLS_AES_C is defined otherwise PSA - * is used. - * - * \warning When using PSA, the user should call `psa_crypto_init()` before - * using any CTR_DRBG operation (except `mbedtls_ctr_drbg_init()`). - * - * \note AES-128 will be used if \c MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH is set. - * - * \note To achieve a 256-bit security strength with CTR_DRBG, - * you must use AES-256 *and* use sufficient entropy. - * See ctr_drbg.h for more details. - * - * Module: library/ctr_drbg.c - * Caller: - * - * Requires: MBEDTLS_AES_C or - * (PSA_WANT_KEY_TYPE_AES and PSA_WANT_ALG_ECB_NO_PADDING and - * MBEDTLS_PSA_CRYPTO_C) - * - * This module provides the CTR_DRBG AES random number generator. - */ -#define MBEDTLS_CTR_DRBG_C - -/** - * \def MBEDTLS_DEBUG_C - * - * Enable the debug functions. - * - * Module: library/debug.c - * Caller: library/ssl_msg.c - * library/ssl_tls.c - * library/ssl_tls12_*.c - * library/ssl_tls13_*.c - * - * This module provides debugging functions. - */ -#define MBEDTLS_DEBUG_C - -/** - * \def MBEDTLS_DES_C - * - * Enable the DES block cipher. - * - * Module: library/des.c - * Caller: library/pem.c - * library/cipher.c - * - * PEM_PARSE uses DES/3DES for decrypting encrypted keys. - * - * \warning DES/3DES are considered weak ciphers and their use constitutes a - * security risk. We recommend considering stronger ciphers instead. - */ -#define MBEDTLS_DES_C - -/** - * \def MBEDTLS_DHM_C - * - * Enable the Diffie-Hellman-Merkle module. - * - * Module: library/dhm.c - * Caller: library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * This module is used by the following key exchanges: - * DHE-RSA, DHE-PSK - * - * \warning Using DHE constitutes a security risk as it - * is not possible to validate custom DH parameters. - * If possible, it is recommended users should consider - * preferring other methods of key exchange. - * See dhm.h for more details. - * - */ -#define MBEDTLS_DHM_C - -/** - * \def MBEDTLS_ECDH_C - * - * Enable the elliptic curve Diffie-Hellman library. - * - * Module: library/ecdh.c - * Caller: library/psa_crypto.c - * library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * This module is used by the following key exchanges: - * ECDHE-ECDSA, ECDHE-RSA, DHE-PSK - * - * Requires: MBEDTLS_ECP_C - */ -#define MBEDTLS_ECDH_C - -/** - * \def MBEDTLS_ECDSA_C - * - * Enable the elliptic curve DSA library. - * - * Module: library/ecdsa.c - * Caller: - * - * This module is used by the following key exchanges: - * ECDHE-ECDSA - * - * Requires: MBEDTLS_ECP_C, MBEDTLS_ASN1_WRITE_C, MBEDTLS_ASN1_PARSE_C, - * and at least one MBEDTLS_ECP_DP_XXX_ENABLED for a - * short Weierstrass curve. - */ -#define MBEDTLS_ECDSA_C - -/** - * \def MBEDTLS_ECJPAKE_C - * - * Enable the elliptic curve J-PAKE library. - * - * \note EC J-PAKE support is based on the Thread v1.0.0 specification. - * It has not been reviewed for compliance with newer standards such as - * Thread v1.1 or RFC 8236. - * - * Module: library/ecjpake.c - * Caller: - * - * This module is used by the following key exchanges: - * ECJPAKE - * - * Requires: MBEDTLS_ECP_C and either MBEDTLS_MD_C or MBEDTLS_PSA_CRYPTO_C - * - * \warning If using a hash that is only provided by PSA drivers, you must - * call psa_crypto_init() before doing any EC J-PAKE operations. - */ -#define MBEDTLS_ECJPAKE_C - -/** - * \def MBEDTLS_ECP_C - * - * Enable the elliptic curve over GF(p) library. - * - * Module: library/ecp.c - * Caller: library/ecdh.c - * library/ecdsa.c - * library/ecjpake.c - * - * Requires: MBEDTLS_BIGNUM_C and at least one MBEDTLS_ECP_DP_XXX_ENABLED - */ -#define MBEDTLS_ECP_C - -/** - * \def MBEDTLS_ENTROPY_C - * - * Enable the platform-specific entropy code. - * - * Module: library/entropy.c - * Caller: - * - * Requires: MBEDTLS_SHA512_C or MBEDTLS_SHA256_C - * - * This module provides a generic entropy pool - */ -#define MBEDTLS_ENTROPY_C - -/** - * \def MBEDTLS_ERROR_C - * - * Enable error code to error string conversion. - * - * Module: library/error.c - * Caller: - * - * This module enables mbedtls_strerror(). - */ -#define MBEDTLS_ERROR_C - -/** - * \def MBEDTLS_GCM_C - * - * Enable the Galois/Counter Mode (GCM). - * - * Module: library/gcm.c - * - * Requires: MBEDTLS_CIPHER_C, MBEDTLS_AES_C or MBEDTLS_CAMELLIA_C or - * MBEDTLS_ARIA_C - * - * This module enables the AES-GCM and CAMELLIA-GCM ciphersuites, if other - * requisites are enabled as well. - */ -#define MBEDTLS_GCM_C - -/** - * \def MBEDTLS_GCM_LARGE_TABLE - * - * Enable large pre-computed tables for Galois/Counter Mode (GCM). - * Can significantly increase throughput on systems without GCM hardware - * acceleration (e.g., AESNI, AESCE). - * - * The mbedtls_gcm_context size will increase by 3840 bytes. - * The code size will increase by roughly 344 bytes. - * - * Module: library/gcm.c - * - * Requires: MBEDTLS_GCM_C - */ -//#define MBEDTLS_GCM_LARGE_TABLE - -/** - * \def MBEDTLS_HKDF_C - * - * Enable the HKDF algorithm (RFC 5869). - * - * Module: library/hkdf.c - * Caller: - * - * Requires: MBEDTLS_MD_C - * - * This module adds support for the Hashed Message Authentication Code - * (HMAC)-based key derivation function (HKDF). - */ -#define MBEDTLS_HKDF_C - -/** - * \def MBEDTLS_HMAC_DRBG_C - * - * Enable the HMAC_DRBG random generator. - * - * Module: library/hmac_drbg.c - * Caller: - * - * Requires: MBEDTLS_MD_C - * - * Uncomment to enable the HMAC_DRBG random number generator. - */ -#define MBEDTLS_HMAC_DRBG_C - -/** - * \def MBEDTLS_LMS_C - * - * Enable the LMS stateful-hash asymmetric signature algorithm. - * - * Module: library/lms.c - * Caller: - * - * Requires: MBEDTLS_PSA_CRYPTO_C - * - * Uncomment to enable the LMS verification algorithm and public key operations. - */ -#define MBEDTLS_LMS_C - -/** - * \def MBEDTLS_LMS_PRIVATE - * - * Enable LMS private-key operations and signing code. Functions enabled by this - * option are experimental, and should not be used in production. - * - * Requires: MBEDTLS_LMS_C - * - * Uncomment to enable the LMS signature algorithm and private key operations. - */ -//#define MBEDTLS_LMS_PRIVATE - -/** - * \def MBEDTLS_NIST_KW_C - * - * Enable the Key Wrapping mode for 128-bit block ciphers, - * as defined in NIST SP 800-38F. Only KW and KWP modes - * are supported. At the moment, only AES is approved by NIST. - * - * Module: library/nist_kw.c - * - * Requires: MBEDTLS_AES_C and MBEDTLS_CIPHER_C - */ -#define MBEDTLS_NIST_KW_C - -/** - * \def MBEDTLS_MD_C - * - * Enable the generic layer for message digest (hashing) and HMAC. - * - * Requires: one of: MBEDTLS_MD5_C, MBEDTLS_RIPEMD160_C, MBEDTLS_SHA1_C, - * MBEDTLS_SHA224_C, MBEDTLS_SHA256_C, MBEDTLS_SHA384_C, - * MBEDTLS_SHA512_C, or MBEDTLS_PSA_CRYPTO_C with at least - * one hash. - * Module: library/md.c - * Caller: library/constant_time.c - * library/ecdsa.c - * library/ecjpake.c - * library/hkdf.c - * library/hmac_drbg.c - * library/pk.c - * library/pkcs5.c - * library/pkcs12.c - * library/psa_crypto_ecp.c - * library/psa_crypto_rsa.c - * library/rsa.c - * library/ssl_cookie.c - * library/ssl_msg.c - * library/ssl_tls.c - * library/x509.c - * library/x509_crt.c - * library/x509write_crt.c - * library/x509write_csr.c - * - * Uncomment to enable generic message digest wrappers. - */ -#define MBEDTLS_MD_C - -/** - * \def MBEDTLS_MD5_C - * - * Enable the MD5 hash algorithm. - * - * Module: library/md5.c - * Caller: library/md.c - * library/pem.c - * library/ssl_tls.c - * - * This module is required for TLS 1.2 depending on the handshake parameters. - * Further, it is used for checking MD5-signed certificates, and for PBKDF1 - * when decrypting PEM-encoded encrypted keys. - * - * \warning MD5 is considered a weak message digest and its use constitutes a - * security risk. If possible, we recommend avoiding dependencies on - * it, and considering stronger message digests instead. - * - */ -#define MBEDTLS_MD5_C - -/** - * \def MBEDTLS_MEMORY_BUFFER_ALLOC_C - * - * Enable the buffer allocator implementation that makes use of a (stack) - * based buffer to 'allocate' dynamic memory. (replaces calloc() and free() - * calls) - * - * Module: library/memory_buffer_alloc.c - * - * Requires: MBEDTLS_PLATFORM_C - * MBEDTLS_PLATFORM_MEMORY (to use it within Mbed TLS) - * - * Enable this module to enable the buffer memory allocator. - */ -//#define MBEDTLS_MEMORY_BUFFER_ALLOC_C - -/** - * \def MBEDTLS_NET_C - * - * Enable the TCP and UDP over IPv6/IPv4 networking routines. - * - * \note This module only works on POSIX/Unix (including Linux, BSD and OS X) - * and Windows. For other platforms, you'll want to disable it, and write your - * own networking callbacks to be passed to \c mbedtls_ssl_set_bio(). - * - * \note See also our Knowledge Base article about porting to a new - * environment: - * https://mbed-tls.readthedocs.io/en/latest/kb/how-to/how-do-i-port-mbed-tls-to-a-new-environment-OS - * - * Module: library/net_sockets.c - * - * This module provides networking routines. - */ -#define MBEDTLS_NET_C - -/** - * \def MBEDTLS_OID_C - * - * Enable the OID database. - * - * Module: library/oid.c - * Caller: library/asn1write.c - * library/pkcs5.c - * library/pkparse.c - * library/pkwrite.c - * library/rsa.c - * library/x509.c - * library/x509_create.c - * library/x509_crl.c - * library/x509_crt.c - * library/x509_csr.c - * library/x509write_crt.c - * library/x509write_csr.c - * - * This modules translates between OIDs and internal values. - */ -#define MBEDTLS_OID_C - -/** - * \def MBEDTLS_PADLOCK_C - * - * Enable VIA Padlock support on x86. - * - * Module: library/padlock.c - * Caller: library/aes.c - * - * Requires: MBEDTLS_HAVE_ASM - * - * This modules adds support for the VIA PadLock on x86. - */ -#define MBEDTLS_PADLOCK_C - -/** - * \def MBEDTLS_PEM_PARSE_C - * - * Enable PEM decoding / parsing. - * - * Module: library/pem.c - * Caller: library/dhm.c - * library/pkparse.c - * library/x509_crl.c - * library/x509_crt.c - * library/x509_csr.c - * - * Requires: MBEDTLS_BASE64_C - * optionally MBEDTLS_MD5_C, or PSA Crypto with MD5 (see below) - * - * \warning When parsing password-protected files, if MD5 is provided only by - * a PSA driver, you must call psa_crypto_init() before the first file. - * - * This modules adds support for decoding / parsing PEM files. - */ -#define MBEDTLS_PEM_PARSE_C - -/** - * \def MBEDTLS_PEM_WRITE_C - * - * Enable PEM encoding / writing. - * - * Module: library/pem.c - * Caller: library/pkwrite.c - * library/x509write_crt.c - * library/x509write_csr.c - * - * Requires: MBEDTLS_BASE64_C - * - * This modules adds support for encoding / writing PEM files. - */ -#define MBEDTLS_PEM_WRITE_C - -/** - * \def MBEDTLS_PK_C - * - * Enable the generic public (asymmetric) key layer. - * - * Module: library/pk.c - * Caller: library/psa_crypto_rsa.c - * library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * library/x509.c - * - * Requires: MBEDTLS_MD_C, MBEDTLS_RSA_C or MBEDTLS_ECP_C - * - * Uncomment to enable generic public key wrappers. - */ -#define MBEDTLS_PK_C - -/** - * \def MBEDTLS_PK_PARSE_C - * - * Enable the generic public (asymmetric) key parser. - * - * Module: library/pkparse.c - * Caller: library/x509_crt.c - * library/x509_csr.c - * - * Requires: MBEDTLS_ASN1_PARSE_C, MBEDTLS_OID_C, MBEDTLS_PK_C - * - * Uncomment to enable generic public key parse functions. - */ -#define MBEDTLS_PK_PARSE_C - -/** - * \def MBEDTLS_PK_WRITE_C - * - * Enable the generic public (asymmetric) key writer. - * - * Module: library/pkwrite.c - * Caller: library/x509write.c - * - * Requires: MBEDTLS_ASN1_WRITE_C, MBEDTLS_OID_C, MBEDTLS_PK_C - * - * Uncomment to enable generic public key write functions. - */ -#define MBEDTLS_PK_WRITE_C - -/** - * \def MBEDTLS_PKCS5_C - * - * Enable PKCS#5 functions. - * - * Module: library/pkcs5.c - * - * Auto-enables: MBEDTLS_MD_C - * - * \warning If using a hash that is only provided by PSA drivers, you must - * call psa_crypto_init() before doing any PKCS5 operations. - * - * This module adds support for the PKCS#5 functions. - */ -#define MBEDTLS_PKCS5_C - -/** - * \def MBEDTLS_PKCS7_C - * - * Enable PKCS #7 core for using PKCS #7-formatted signatures. - * RFC Link - https://tools.ietf.org/html/rfc2315 - * - * Module: library/pkcs7.c - * - * Requires: MBEDTLS_ASN1_PARSE_C, MBEDTLS_OID_C, MBEDTLS_PK_PARSE_C, - * MBEDTLS_X509_CRT_PARSE_C MBEDTLS_X509_CRL_PARSE_C, - * MBEDTLS_BIGNUM_C, MBEDTLS_MD_C - * - * This module is required for the PKCS #7 parsing modules. - */ -#define MBEDTLS_PKCS7_C - -/** - * \def MBEDTLS_PKCS12_C - * - * Enable PKCS#12 PBE functions. - * Adds algorithms for parsing PKCS#8 encrypted private keys - * - * Module: library/pkcs12.c - * Caller: library/pkparse.c - * - * Requires: MBEDTLS_ASN1_PARSE_C and either MBEDTLS_MD_C or - * MBEDTLS_PSA_CRYPTO_C. - * - * \warning If using a hash that is only provided by PSA drivers, you must - * call psa_crypto_init() before doing any PKCS12 operations. - * - * This module enables PKCS#12 functions. - */ -#define MBEDTLS_PKCS12_C - -/** - * \def MBEDTLS_PLATFORM_C - * - * Enable the platform abstraction layer that allows you to re-assign - * functions like calloc(), free(), snprintf(), printf(), fprintf(), exit(). - * - * Enabling MBEDTLS_PLATFORM_C enables to use of MBEDTLS_PLATFORM_XXX_ALT - * or MBEDTLS_PLATFORM_XXX_MACRO directives, allowing the functions mentioned - * above to be specified at runtime or compile time respectively. - * - * \note This abstraction layer must be enabled on Windows (including MSYS2) - * as other modules rely on it for a fixed snprintf implementation. - * - * Module: library/platform.c - * Caller: Most other .c files - * - * This module enables abstraction of common (libc) functions. - */ -#define MBEDTLS_PLATFORM_C - -/** - * \def MBEDTLS_POLY1305_C - * - * Enable the Poly1305 MAC algorithm. - * - * Module: library/poly1305.c - * Caller: library/chachapoly.c - */ -#define MBEDTLS_POLY1305_C - -/** - * \def MBEDTLS_PSA_CRYPTO_C - * - * Enable the Platform Security Architecture (PSA) cryptography API. - * - * \note In multithreaded applications, you must enable #MBEDTLS_THREADING_C, - * unless only one thread ever calls `psa_xxx()` functions. - * That includes indirect calls, such as: - * - performing a TLS handshake if support for TLS 1.3 is enabled; - * - using a TLS 1.3 connection; - * - indirect calls from PK, X.509 or SSL functions when - * #MBEDTLS_USE_PSA_CRYPTO is enabled; - * - indirect calls to calculate a hash when #MBEDTLS_MD_C is disabled; - * - any other call to a function that requires calling psa_crypto_init() - * beforehand. - * - * Module: library/psa_crypto.c - * - * Requires: either MBEDTLS_CTR_DRBG_C and MBEDTLS_ENTROPY_C, - * or MBEDTLS_HMAC_DRBG_C and MBEDTLS_ENTROPY_C, - * or MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG. - * Auto-enables: MBEDTLS_CIPHER_C if any unauthenticated (ie, non-AEAD) cipher - * is enabled in PSA (unless it's fully accelerated, see - * docs/driver-only-builds.md about that). - */ -#define MBEDTLS_PSA_CRYPTO_C - -/** - * \def MBEDTLS_PSA_CRYPTO_SE_C - * - * Enable dynamic secure element support in the Platform Security Architecture - * cryptography API. - * - * \deprecated This feature is deprecated. Please switch to the PSA driver - * interface. - * - * \warning This feature is not thread-safe, and should not be used in a - * multi-threaded environment. - * - * Module: library/psa_crypto_se.c - * - * Requires: MBEDTLS_PSA_CRYPTO_C, MBEDTLS_PSA_CRYPTO_STORAGE_C - * - */ -//#define MBEDTLS_PSA_CRYPTO_SE_C - -/** - * \def MBEDTLS_PSA_CRYPTO_STORAGE_C - * - * Enable the Platform Security Architecture persistent key storage. - * - * Module: library/psa_crypto_storage.c - * - * Requires: MBEDTLS_PSA_CRYPTO_C, - * either MBEDTLS_PSA_ITS_FILE_C or a native implementation of - * the PSA ITS interface - */ -#define MBEDTLS_PSA_CRYPTO_STORAGE_C - -/** - * \def MBEDTLS_PSA_ITS_FILE_C - * - * Enable the emulation of the Platform Security Architecture - * Internal Trusted Storage (PSA ITS) over files. - * - * Module: library/psa_its_file.c - * - * Requires: MBEDTLS_FS_IO - */ -#define MBEDTLS_PSA_ITS_FILE_C - -/** - * \def MBEDTLS_PSA_STATIC_KEY_SLOTS - * - * Statically preallocate memory to store keys' material in PSA instead - * of allocating it dynamically when required. This allows builds without a - * heap, if none of the enabled cryptographic implementations or other features - * require it. - * This feature affects both volatile and persistent keys which means that - * it's not possible to persistently store a key which is larger than - * #MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE. - * - * \note This feature comes with a (potentially) higher RAM usage since: - * - All the key slots are allocated no matter if they are used or not. - * - Each key buffer's length is #MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE bytes. - * - * Requires: MBEDTLS_PSA_CRYPTO_C - * - */ -//#define MBEDTLS_PSA_STATIC_KEY_SLOTS - -/** - * \def MBEDTLS_RIPEMD160_C - * - * Enable the RIPEMD-160 hash algorithm. - * - * Module: library/ripemd160.c - * Caller: library/md.c - * - */ -#define MBEDTLS_RIPEMD160_C - -/** - * \def MBEDTLS_RSA_C - * - * Enable the RSA public-key cryptosystem. - * - * Module: library/rsa.c - * library/rsa_alt_helpers.c - * Caller: library/pk.c - * library/psa_crypto.c - * library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * This module is used by the following key exchanges: - * RSA, DHE-RSA, ECDHE-RSA, RSA-PSK - * - * Requires: MBEDTLS_BIGNUM_C, MBEDTLS_OID_C - */ -#define MBEDTLS_RSA_C - -/** - * \def MBEDTLS_SHA1_C - * - * Enable the SHA1 cryptographic hash algorithm. - * - * Module: library/sha1.c - * Caller: library/md.c - * library/psa_crypto_hash.c - * - * This module is required for TLS 1.2 depending on the handshake parameters, - * and for SHA1-signed certificates. - * - * \warning SHA-1 is considered a weak message digest and its use constitutes - * a security risk. If possible, we recommend avoiding dependencies - * on it, and considering stronger message digests instead. - * - */ -#define MBEDTLS_SHA1_C - -/** - * \def MBEDTLS_SHA224_C - * - * Enable the SHA-224 cryptographic hash algorithm. - * - * Module: library/sha256.c - * Caller: library/md.c - * library/ssl_cookie.c - * - * This module adds support for SHA-224. - */ -#define MBEDTLS_SHA224_C - -/** - * \def MBEDTLS_SHA256_C - * - * Enable the SHA-256 cryptographic hash algorithm. - * - * Module: library/sha256.c - * Caller: library/entropy.c - * library/md.c - * library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * This module adds support for SHA-256. - * This module is required for the SSL/TLS 1.2 PRF function. - */ -#define MBEDTLS_SHA256_C - -/** - * \def MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT - * - * Enable acceleration of the SHA-256 and SHA-224 cryptographic hash algorithms - * with the ARMv8 cryptographic extensions if they are available at runtime. - * If not, the library will fall back to the C implementation. - * - * \note If MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT is defined when building - * for a non-Armv8-A build it will be silently ignored. - * - * \note Minimum compiler versions for this feature are Clang 4.0, - * armclang 6.6 or GCC 6.0. - * - * \note \c CFLAGS must be set to a minimum of \c -march=armv8-a+crypto for - * armclang <= 6.9 - * - * \note This was previously known as MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT. - * That name is deprecated, but may still be used as an alternative form for this - * option. - * - * \warning MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT cannot be defined at the - * same time as MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY. - * - * Requires: MBEDTLS_SHA256_C. - * - * Module: library/sha256.c - * - * Uncomment to have the library check for the Armv8-A SHA-256 crypto extensions - * and use them if available. - */ -//#define MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT - -/** - * \def MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT - * - * \deprecated This is now known as MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT. - * This name is now deprecated, but may still be used as an alternative form for - * this option. - */ -//#define MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT - -/** - * \def MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY - * - * Enable acceleration of the SHA-256 and SHA-224 cryptographic hash algorithms - * with the ARMv8 cryptographic extensions, which must be available at runtime - * or else an illegal instruction fault will occur. - * - * \note This allows builds with a smaller code size than with - * MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT - * - * \note Minimum compiler versions for this feature are Clang 4.0, - * armclang 6.6 or GCC 6.0. - * - * \note \c CFLAGS must be set to a minimum of \c -march=armv8-a+crypto for - * armclang <= 6.9 - * - * \note This was previously known as MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY. - * That name is deprecated, but may still be used as an alternative form for this - * option. - * - * \warning MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY cannot be defined at the same - * time as MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT. - * - * Requires: MBEDTLS_SHA256_C. - * - * Module: library/sha256.c - * - * Uncomment to have the library use the Armv8-A SHA-256 crypto extensions - * unconditionally. - */ -//#define MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY - -/** - * \def MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY - * - * \deprecated This is now known as MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY. - * This name is now deprecated, but may still be used as an alternative form for - * this option. - */ -//#define MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY - -/** - * \def MBEDTLS_SHA384_C - * - * Enable the SHA-384 cryptographic hash algorithm. - * - * Module: library/sha512.c - * Caller: library/md.c - * library/psa_crypto_hash.c - * library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * Comment to disable SHA-384 - */ -#define MBEDTLS_SHA384_C - -/** - * \def MBEDTLS_SHA512_C - * - * Enable SHA-512 cryptographic hash algorithms. - * - * Module: library/sha512.c - * Caller: library/entropy.c - * library/md.c - * library/ssl_tls.c - * library/ssl_cookie.c - * - * This module adds support for SHA-512. - */ -#define MBEDTLS_SHA512_C - -/** - * \def MBEDTLS_SHA3_C - * - * Enable the SHA3 cryptographic hash algorithm. - * - * Module: library/sha3.c - * - * This module adds support for SHA3. - */ -#define MBEDTLS_SHA3_C - -/** - * \def MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT - * - * Enable acceleration of the SHA-512 and SHA-384 cryptographic hash algorithms - * with the ARMv8 cryptographic extensions if they are available at runtime. - * If not, the library will fall back to the C implementation. - * - * \note If MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT is defined when building - * for a non-Aarch64 build it will be silently ignored. - * - * \note Minimum compiler versions for this feature are Clang 7.0, - * armclang 6.9 or GCC 8.0. - * - * \note \c CFLAGS must be set to a minimum of \c -march=armv8.2-a+sha3 for - * armclang 6.9 - * - * \warning MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT cannot be defined at the - * same time as MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY. - * - * Requires: MBEDTLS_SHA512_C. - * - * Module: library/sha512.c - * - * Uncomment to have the library check for the A64 SHA-512 crypto extensions - * and use them if available. - */ -//#define MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT - -/** - * \def MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY - * - * Enable acceleration of the SHA-512 and SHA-384 cryptographic hash algorithms - * with the ARMv8 cryptographic extensions, which must be available at runtime - * or else an illegal instruction fault will occur. - * - * \note This allows builds with a smaller code size than with - * MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT - * - * \note Minimum compiler versions for this feature are Clang 7.0, - * armclang 6.9 or GCC 8.0. - * - * \note \c CFLAGS must be set to a minimum of \c -march=armv8.2-a+sha3 for - * armclang 6.9 - * - * \warning MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY cannot be defined at the same - * time as MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT. - * - * Requires: MBEDTLS_SHA512_C. - * - * Module: library/sha512.c - * - * Uncomment to have the library use the A64 SHA-512 crypto extensions - * unconditionally. - */ -//#define MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY - -/** - * \def MBEDTLS_SSL_CACHE_C - * - * Enable simple SSL cache implementation. - * - * Module: library/ssl_cache.c - * Caller: - * - * Requires: MBEDTLS_SSL_CACHE_C - */ -#define MBEDTLS_SSL_CACHE_C - -/** - * \def MBEDTLS_SSL_COOKIE_C - * - * Enable basic implementation of DTLS cookies for hello verification. - * - * Module: library/ssl_cookie.c - * Caller: - */ -#define MBEDTLS_SSL_COOKIE_C - -/** - * \def MBEDTLS_SSL_TICKET_C - * - * Enable an implementation of TLS server-side callbacks for session tickets. - * - * Module: library/ssl_ticket.c - * Caller: - * - * Requires: (MBEDTLS_CIPHER_C || MBEDTLS_USE_PSA_CRYPTO) && - * (MBEDTLS_GCM_C || MBEDTLS_CCM_C || MBEDTLS_CHACHAPOLY_C) - */ -#define MBEDTLS_SSL_TICKET_C - -/** - * \def MBEDTLS_SSL_CLI_C - * - * Enable the SSL/TLS client code. - * - * Module: library/ssl*_client.c - * Caller: - * - * Requires: MBEDTLS_SSL_TLS_C - * - * This module is required for SSL/TLS client support. - */ -#define MBEDTLS_SSL_CLI_C - -/** - * \def MBEDTLS_SSL_SRV_C - * - * Enable the SSL/TLS server code. - * - * Module: library/ssl*_server.c - * Caller: - * - * Requires: MBEDTLS_SSL_TLS_C - * - * This module is required for SSL/TLS server support. - */ -#define MBEDTLS_SSL_SRV_C - -/** - * \def MBEDTLS_SSL_TLS_C - * - * Enable the generic SSL/TLS code. - * - * Module: library/ssl_tls.c - * Caller: library/ssl*_client.c - * library/ssl*_server.c - * - * Requires: MBEDTLS_CIPHER_C, MBEDTLS_MD_C - * and at least one of the MBEDTLS_SSL_PROTO_XXX defines - * - * This module is required for SSL/TLS. - */ -#define MBEDTLS_SSL_TLS_C - -/** - * \def MBEDTLS_THREADING_C - * - * Enable the threading abstraction layer. - * - * Traditionally, Mbed TLS assumes it is used in a non-threaded environment or - * that contexts are not shared between threads. If you do intend to use contexts - * between threads, you will need to enable this layer to prevent race - * conditions. - * - * The PSA subsystem has an implicit shared context. Therefore, you must - * enable this option if more than one thread may use any part of - * Mbed TLS that is implemented on top of the PSA subsystem. - * - * You must enable this option in multithreaded applications where more than - * one thread performs any of the following operations: - * - * - Any call to a PSA function (`psa_xxx()`). - * - Any call to a TLS, X.509 or PK function (`mbedtls_ssl_xxx()`, - * `mbedtls_x509_xxx()`, `mbedtls_pkcs7_xxx()`, `mbedtls_pk_xxx()`) - * if `MBEDTLS_USE_PSA_CRYPTO` is enabled (regardless of whether individual - * TLS, X.509 or PK contexts are shared between threads). - * - A TLS 1.3 connection, regardless of the compile-time configuration. - * - Any library feature that calculates a hash, if `MBEDTLS_MD_C` is disabled. - * As an exception, algorithm-specific low-level modules do not require - * threading protection unless the contexts are shared between threads. - * - Any library feature that performs symmetric encryption or decryption, - * if `MBEDTLS_CIPHER_C` is disabled. - * As an exception, algorithm-specific low-level modules do not require - * threading protection unless the contexts are shared between threads. - * - Any use of a cryptographic context if the same context is used in - * multiple threads. - * - Any call to a function where the documentation specifies that - * psa_crypto_init() must be called prior to that function. - * - * See also our Knowledge Base article about threading: - * https://mbed-tls.readthedocs.io/en/latest/kb/development/thread-safety-and-multi-threading - * - * Module: library/threading.c - * - * This allows different threading implementations (self-implemented or - * provided). - * - * You will have to enable either MBEDTLS_THREADING_ALT or - * MBEDTLS_THREADING_PTHREAD. - * - * Enable this layer to allow use of mutexes within Mbed TLS - */ -//#define MBEDTLS_THREADING_C - -/** - * \def MBEDTLS_TIMING_C - * - * Enable the semi-portable timing interface. - * - * \note The provided implementation only works on POSIX/Unix (including Linux, - * BSD and OS X) and Windows. On other platforms, you can either disable that - * module and provide your own implementations of the callbacks needed by - * \c mbedtls_ssl_set_timer_cb() for DTLS, or leave it enabled and provide - * your own implementation of the whole module by setting - * \c MBEDTLS_TIMING_ALT in the current file. - * - * \note The timing module will include time.h on suitable platforms - * regardless of the setting of MBEDTLS_HAVE_TIME, unless - * MBEDTLS_TIMING_ALT is used. See timing.c for more information. - * - * \note See also our Knowledge Base article about porting to a new - * environment: - * https://mbed-tls.readthedocs.io/en/latest/kb/how-to/how-do-i-port-mbed-tls-to-a-new-environment-OS - * - * Module: library/timing.c - */ -#define MBEDTLS_TIMING_C - -/** - * \def MBEDTLS_VERSION_C - * - * Enable run-time version information. - * - * Module: library/version.c - * - * This module provides run-time version information. - */ -#define MBEDTLS_VERSION_C - -/** - * \def MBEDTLS_X509_USE_C - * - * Enable X.509 core for using certificates. - * - * Module: library/x509.c - * Caller: library/x509_crl.c - * library/x509_crt.c - * library/x509_csr.c - * - * Requires: MBEDTLS_ASN1_PARSE_C, MBEDTLS_BIGNUM_C, MBEDTLS_OID_C, MBEDTLS_PK_PARSE_C, - * (MBEDTLS_MD_C or MBEDTLS_USE_PSA_CRYPTO) - * - * \warning If building with MBEDTLS_USE_PSA_CRYPTO, you must call - * psa_crypto_init() before doing any X.509 operation. - * - * This module is required for the X.509 parsing modules. - */ -#define MBEDTLS_X509_USE_C - -/** - * \def MBEDTLS_X509_CRT_PARSE_C - * - * Enable X.509 certificate parsing. - * - * Module: library/x509_crt.c - * Caller: library/ssl_tls.c - * library/ssl*_client.c - * library/ssl*_server.c - * - * Requires: MBEDTLS_X509_USE_C - * - * This module is required for X.509 certificate parsing. - */ -#define MBEDTLS_X509_CRT_PARSE_C - -/** - * \def MBEDTLS_X509_CRL_PARSE_C - * - * Enable X.509 CRL parsing. - * - * Module: library/x509_crl.c - * Caller: library/x509_crt.c - * - * Requires: MBEDTLS_X509_USE_C - * - * This module is required for X.509 CRL parsing. - */ -#define MBEDTLS_X509_CRL_PARSE_C - -/** - * \def MBEDTLS_X509_CSR_PARSE_C - * - * Enable X.509 Certificate Signing Request (CSR) parsing. - * - * Module: library/x509_csr.c - * Caller: library/x509_crt_write.c - * - * Requires: MBEDTLS_X509_USE_C - * - * This module is used for reading X.509 certificate request. - */ -#define MBEDTLS_X509_CSR_PARSE_C - -/** - * \def MBEDTLS_X509_CREATE_C - * - * Enable X.509 core for creating certificates. - * - * Module: library/x509_create.c - * - * Requires: MBEDTLS_BIGNUM_C, MBEDTLS_OID_C, MBEDTLS_PK_PARSE_C, - * (MBEDTLS_MD_C or MBEDTLS_USE_PSA_CRYPTO) - * - * \warning If building with MBEDTLS_USE_PSA_CRYPTO, you must call - * psa_crypto_init() before doing any X.509 create operation. - * - * This module is the basis for creating X.509 certificates and CSRs. - */ -#define MBEDTLS_X509_CREATE_C - -/** - * \def MBEDTLS_X509_CRT_WRITE_C - * - * Enable creating X.509 certificates. - * - * Module: library/x509_crt_write.c - * - * Requires: MBEDTLS_X509_CREATE_C - * - * This module is required for X.509 certificate creation. - */ -#define MBEDTLS_X509_CRT_WRITE_C - -/** - * \def MBEDTLS_X509_CSR_WRITE_C - * - * Enable creating X.509 Certificate Signing Requests (CSR). - * - * Module: library/x509_csr_write.c - * - * Requires: MBEDTLS_X509_CREATE_C - * - * This module is required for X.509 certificate request writing. - */ -#define MBEDTLS_X509_CSR_WRITE_C - -/** \} name SECTION: Mbed TLS modules */ - -/** - * \name SECTION: General configuration options - * - * This section contains Mbed TLS build settings that are not associated - * with a particular module. - * - * \{ - */ - -/** - * \def MBEDTLS_CONFIG_FILE - * - * If defined, this is a header which will be included instead of - * `"mbedtls/mbedtls_config.h"`. - * This header file specifies the compile-time configuration of Mbed TLS. - * Unlike other configuration options, this one must be defined on the - * compiler command line: a definition in `mbedtls_config.h` would have - * no effect. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_CONFIG_FILE "mbedtls/mbedtls_config.h" - -/** - * \def MBEDTLS_USER_CONFIG_FILE - * - * If defined, this is a header which will be included after - * `"mbedtls/mbedtls_config.h"` or #MBEDTLS_CONFIG_FILE. - * This allows you to modify the default configuration, including the ability - * to undefine options that are enabled by default. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_USER_CONFIG_FILE "/dev/null" - -/** - * \def MBEDTLS_PSA_CRYPTO_CONFIG_FILE - * - * If defined, this is a header which will be included instead of - * `"psa/crypto_config.h"`. - * This header file specifies which cryptographic mechanisms are available - * through the PSA API when #MBEDTLS_PSA_CRYPTO_CONFIG is enabled, and - * is not used when #MBEDTLS_PSA_CRYPTO_CONFIG is disabled. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_PSA_CRYPTO_CONFIG_FILE "psa/crypto_config.h" - -/** - * \def MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE - * - * If defined, this is a header which will be included after - * `"psa/crypto_config.h"` or #MBEDTLS_PSA_CRYPTO_CONFIG_FILE. - * This allows you to modify the default configuration, including the ability - * to undefine options that are enabled by default. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_PSA_CRYPTO_USER_CONFIG_FILE "/dev/null" - -/** - * \def MBEDTLS_PSA_CRYPTO_PLATFORM_FILE - * - * If defined, this is a header which will be included instead of - * `"psa/crypto_platform.h"`. This file should declare the same identifiers - * as the one in Mbed TLS, but with definitions adapted to the platform on - * which the library code will run. - * - * \note The required content of this header can vary from one version of - * Mbed TLS to the next. Integrators who provide an alternative file - * should review the changes in the original file whenever they - * upgrade Mbed TLS. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_PSA_CRYPTO_PLATFORM_FILE "psa/crypto_platform_alt.h" - -/** - * \def MBEDTLS_PSA_CRYPTO_STRUCT_FILE - * - * If defined, this is a header which will be included instead of - * `"psa/crypto_struct.h"`. This file should declare the same identifiers - * as the one in Mbed TLS, but with definitions adapted to the environment - * in which the library code will run. The typical use for this feature - * is to provide alternative type definitions on the client side in - * client-server integrations of PSA crypto, where operation structures - * contain handles instead of cryptographic data. - * - * \note The required content of this header can vary from one version of - * Mbed TLS to the next. Integrators who provide an alternative file - * should review the changes in the original file whenever they - * upgrade Mbed TLS. - * - * This macro is expanded after an \#include directive. This is a popular but - * non-standard feature of the C language, so this feature is only available - * with compilers that perform macro expansion on an \#include line. - * - * The value of this symbol is typically a path in double quotes, either - * absolute or relative to a directory on the include search path. - */ -//#define MBEDTLS_PSA_CRYPTO_STRUCT_FILE "psa/crypto_struct_alt.h" - -/** \} name SECTION: General configuration options */ - -/** - * \name SECTION: Module configuration options - * - * This section allows for the setting of module specific sizes and - * configuration options. The default values are already present in the - * relevant header files and should suffice for the regular use cases. - * - * Our advice is to enable options and change their values here - * only if you have a good reason and know the consequences. - * \{ - */ -/* The Doxygen documentation here is used when a user comments out a - * setting and runs doxygen themselves. On the other hand, when we typeset - * the full documentation including disabled settings, the documentation - * in specific modules' header files is used if present. When editing this - * file, make sure that each option is documented in exactly one place, - * plus optionally a same-line Doxygen comment here if there is a Doxygen - * comment in the specific module. */ - -/* MPI / BIGNUM options */ -//#define MBEDTLS_MPI_WINDOW_SIZE 2 /**< Maximum window size used. */ -//#define MBEDTLS_MPI_MAX_SIZE 1024 /**< Maximum number of bytes for usable MPIs. */ - -/* CTR_DRBG options */ -//#define MBEDTLS_CTR_DRBG_ENTROPY_LEN 48 /**< Amount of entropy used per seed by default (48 with SHA-512, 32 with SHA-256) */ -//#define MBEDTLS_CTR_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ -//#define MBEDTLS_CTR_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ -//#define MBEDTLS_CTR_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -//#define MBEDTLS_CTR_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - -/* HMAC_DRBG options */ -//#define MBEDTLS_HMAC_DRBG_RESEED_INTERVAL 10000 /**< Interval before reseed is performed by default */ -//#define MBEDTLS_HMAC_DRBG_MAX_INPUT 256 /**< Maximum number of additional input bytes */ -//#define MBEDTLS_HMAC_DRBG_MAX_REQUEST 1024 /**< Maximum number of requested bytes per call */ -//#define MBEDTLS_HMAC_DRBG_MAX_SEED_INPUT 384 /**< Maximum size of (re)seed buffer */ - -/* ECP options */ -//#define MBEDTLS_ECP_WINDOW_SIZE 4 /**< Maximum window size used */ -//#define MBEDTLS_ECP_FIXED_POINT_OPTIM 1 /**< Enable fixed-point speed-up */ - -/* Entropy options */ -//#define MBEDTLS_ENTROPY_MAX_SOURCES 20 /**< Maximum number of sources supported */ -//#define MBEDTLS_ENTROPY_MAX_GATHER 128 /**< Maximum amount requested from entropy sources */ -//#define MBEDTLS_ENTROPY_MIN_HARDWARE 32 /**< Default minimum number of bytes required for the hardware entropy source mbedtls_hardware_poll() before entropy is released */ - -/* Memory buffer allocator options */ -//#define MBEDTLS_MEMORY_ALIGN_MULTIPLE 4 /**< Align on multiples of this value */ - -/* Platform options */ -//#define MBEDTLS_PLATFORM_STD_MEM_HDR /**< Header to include if MBEDTLS_PLATFORM_NO_STD_FUNCTIONS is defined. Don't define if no header is needed. */ - -/** \def MBEDTLS_PLATFORM_STD_CALLOC - * - * Default allocator to use, can be undefined. - * It must initialize the allocated buffer memory to zeroes. - * The size of the buffer is the product of the two parameters. - * The calloc function returns either a null pointer or a pointer to the allocated space. - * If the product is 0, the function may either return NULL or a valid pointer to an array of size 0 which is a valid input to the deallocation function. - * An uninitialized #MBEDTLS_PLATFORM_STD_CALLOC always fails, returning a null pointer. - * See the description of #MBEDTLS_PLATFORM_MEMORY for more details. - * The corresponding deallocation function is #MBEDTLS_PLATFORM_STD_FREE. - */ -//#define MBEDTLS_PLATFORM_STD_CALLOC calloc - -/** \def MBEDTLS_PLATFORM_STD_FREE - * - * Default free to use, can be undefined. - * NULL is a valid parameter, and the function must do nothing. - * A non-null parameter will always be a pointer previously returned by #MBEDTLS_PLATFORM_STD_CALLOC and not yet freed. - * An uninitialized #MBEDTLS_PLATFORM_STD_FREE does not do anything. - * See the description of #MBEDTLS_PLATFORM_MEMORY for more details (same principles as for MBEDTLS_PLATFORM_STD_CALLOC apply). - */ -//#define MBEDTLS_PLATFORM_STD_FREE free -//#define MBEDTLS_PLATFORM_STD_SETBUF setbuf /**< Default setbuf to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_EXIT exit /**< Default exit to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_TIME time /**< Default time to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled */ -//#define MBEDTLS_PLATFORM_STD_FPRINTF fprintf /**< Default fprintf to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_PRINTF printf /**< Default printf to use, can be undefined */ -/* Note: your snprintf must correctly zero-terminate the buffer! */ -//#define MBEDTLS_PLATFORM_STD_SNPRINTF snprintf /**< Default snprintf to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_EXIT_SUCCESS 0 /**< Default exit value to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_EXIT_FAILURE 1 /**< Default exit value to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_NV_SEED_READ mbedtls_platform_std_nv_seed_read /**< Default nv_seed_read function to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_NV_SEED_WRITE mbedtls_platform_std_nv_seed_write /**< Default nv_seed_write function to use, can be undefined */ -//#define MBEDTLS_PLATFORM_STD_NV_SEED_FILE "seedfile" /**< Seed file to read/write with default implementation */ - -/* To use the following function macros, MBEDTLS_PLATFORM_C must be enabled. */ -/* MBEDTLS_PLATFORM_XXX_MACRO and MBEDTLS_PLATFORM_XXX_ALT cannot both be defined */ -//#define MBEDTLS_PLATFORM_CALLOC_MACRO calloc /**< Default allocator macro to use, can be undefined. See MBEDTLS_PLATFORM_STD_CALLOC for requirements. */ -//#define MBEDTLS_PLATFORM_FREE_MACRO free /**< Default free macro to use, can be undefined. See MBEDTLS_PLATFORM_STD_FREE for requirements. */ -//#define MBEDTLS_PLATFORM_EXIT_MACRO exit /**< Default exit macro to use, can be undefined */ -//#define MBEDTLS_PLATFORM_SETBUF_MACRO setbuf /**< Default setbuf macro to use, can be undefined */ -//#define MBEDTLS_PLATFORM_TIME_MACRO time /**< Default time macro to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled */ -//#define MBEDTLS_PLATFORM_TIME_TYPE_MACRO time_t /**< Default time macro to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled */ -//#define MBEDTLS_PLATFORM_FPRINTF_MACRO fprintf /**< Default fprintf macro to use, can be undefined */ -//#define MBEDTLS_PLATFORM_PRINTF_MACRO printf /**< Default printf macro to use, can be undefined */ -/* Note: your snprintf must correctly zero-terminate the buffer! */ -//#define MBEDTLS_PLATFORM_SNPRINTF_MACRO snprintf /**< Default snprintf macro to use, can be undefined */ -//#define MBEDTLS_PLATFORM_VSNPRINTF_MACRO vsnprintf /**< Default vsnprintf macro to use, can be undefined */ -//#define MBEDTLS_PLATFORM_NV_SEED_READ_MACRO mbedtls_platform_std_nv_seed_read /**< Default nv_seed_read function to use, can be undefined */ -//#define MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO mbedtls_platform_std_nv_seed_write /**< Default nv_seed_write function to use, can be undefined */ -//#define MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO int64_t //#define MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO int64_t /**< Default milliseconds time macro to use, can be undefined. MBEDTLS_HAVE_TIME must be enabled. It must be signed, and at least 64 bits. If it is changed from the default, MBEDTLS_PRINTF_MS_TIME must be updated to match.*/ -//#define MBEDTLS_PRINTF_MS_TIME PRId64 /**< Default fmt for printf. That's avoid compiler warning if mbedtls_ms_time_t is redefined */ - -/** \def MBEDTLS_CHECK_RETURN - * - * This macro is used at the beginning of the declaration of a function - * to indicate that its return value should be checked. It should - * instruct the compiler to emit a warning or an error if the function - * is called without checking its return value. - * - * There is a default implementation for popular compilers in platform_util.h. - * You can override the default implementation by defining your own here. - * - * If the implementation here is empty, this will effectively disable the - * checking of functions' return values. - */ -//#define MBEDTLS_CHECK_RETURN __attribute__((__warn_unused_result__)) - -/** \def MBEDTLS_IGNORE_RETURN - * - * This macro requires one argument, which should be a C function call. - * If that function call would cause a #MBEDTLS_CHECK_RETURN warning, this - * warning is suppressed. - */ -//#define MBEDTLS_IGNORE_RETURN( result ) ((void) !(result)) - -/* PSA options */ -/** - * Use HMAC_DRBG with the specified hash algorithm for HMAC_DRBG for the - * PSA crypto subsystem. - * - * If this option is unset, the library chooses a hash (currently between - * #MBEDTLS_MD_SHA512 and #MBEDTLS_MD_SHA256) based on availability and - * unspecified heuristics. - * - * \note The PSA crypto subsystem uses the first available mechanism amongst - * the following: - * - #MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG if enabled; - * - Entropy from #MBEDTLS_ENTROPY_C plus CTR_DRBG with AES - * if #MBEDTLS_CTR_DRBG_C is enabled; - * - Entropy from #MBEDTLS_ENTROPY_C plus HMAC_DRBG. - * - * A future version may reevaluate the prioritization of DRBG mechanisms. - */ -//#define MBEDTLS_PSA_HMAC_DRBG_MD_TYPE MBEDTLS_MD_SHA256 - -/** \def MBEDTLS_PSA_KEY_SLOT_COUNT - * - * When #MBEDTLS_PSA_KEY_STORE_DYNAMIC is disabled, - * the maximum amount of PSA keys simultaneously in memory. This counts all - * volatile keys, plus loaded persistent keys. - * - * When #MBEDTLS_PSA_KEY_STORE_DYNAMIC is enabled, - * the maximum number of loaded persistent keys. - * - * Currently, persistent keys do not need to be loaded all the time while - * a multipart operation is in progress, only while the operation is being - * set up. This may change in future versions of the library. - * - * Currently, the library traverses of the whole table on each access to a - * persistent key. Therefore large values may cause poor performance. - * - * This option has no effect when #MBEDTLS_PSA_CRYPTO_C is disabled. - */ -//#define MBEDTLS_PSA_KEY_SLOT_COUNT 32 - -/** - * \def MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE - * - * Define the size (in bytes) of each static key buffer when - * #MBEDTLS_PSA_STATIC_KEY_SLOTS is set. If not - * explicitly defined then it's automatically guessed from available PSA keys - * enabled in the build through PSA_WANT_xxx symbols. - * If required by the application this parameter can be set to higher values - * in order to store larger objects (ex: raw keys), but please note that this - * will increase RAM usage. - */ -//#define MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE 256 - -/* RSA OPTIONS */ -//#define MBEDTLS_RSA_GEN_KEY_MIN_BITS 1024 /**< Minimum RSA key size that can be generated in bits (Minimum possible value is 128 bits) */ - -/* SSL Cache options */ -//#define MBEDTLS_SSL_CACHE_DEFAULT_TIMEOUT 86400 /**< 1 day */ -//#define MBEDTLS_SSL_CACHE_DEFAULT_MAX_ENTRIES 50 /**< Maximum entries in cache */ - -/* SSL options */ - -/** \def MBEDTLS_SSL_IN_CONTENT_LEN - * - * Maximum length (in bytes) of incoming plaintext fragments. - * - * This determines the size of the incoming TLS I/O buffer in such a way - * that it is capable of holding the specified amount of plaintext data, - * regardless of the protection mechanism used. - * - * \note When using a value less than the default of 16KB on the client, it is - * recommended to use the Maximum Fragment Length (MFL) extension to - * inform the server about this limitation. On the server, there - * is no supported, standardized way of informing the client about - * restriction on the maximum size of incoming messages, and unless - * the limitation has been communicated by other means, it is recommended - * to only change the outgoing buffer size #MBEDTLS_SSL_OUT_CONTENT_LEN - * while keeping the default value of 16KB for the incoming buffer. - * - * Uncomment to set the maximum plaintext size of the incoming I/O buffer. - */ -//#define MBEDTLS_SSL_IN_CONTENT_LEN 16384 - -/** \def MBEDTLS_SSL_CID_IN_LEN_MAX - * - * The maximum length of CIDs used for incoming DTLS messages. - * - */ -//#define MBEDTLS_SSL_CID_IN_LEN_MAX 32 - -/** \def MBEDTLS_SSL_CID_OUT_LEN_MAX - * - * The maximum length of CIDs used for outgoing DTLS messages. - * - */ -//#define MBEDTLS_SSL_CID_OUT_LEN_MAX 32 - -/** \def MBEDTLS_SSL_CID_TLS1_3_PADDING_GRANULARITY - * - * This option controls the use of record plaintext padding - * in TLS 1.3 and when using the Connection ID extension in DTLS 1.2. - * - * The padding will always be chosen so that the length of the - * padded plaintext is a multiple of the value of this option. - * - * Note: A value of \c 1 means that no padding will be used - * for outgoing records. - * - * Note: On systems lacking division instructions, - * a power of two should be preferred. - */ -//#define MBEDTLS_SSL_CID_TLS1_3_PADDING_GRANULARITY 16 - -/** \def MBEDTLS_SSL_OUT_CONTENT_LEN - * - * Maximum length (in bytes) of outgoing plaintext fragments. - * - * This determines the size of the outgoing TLS I/O buffer in such a way - * that it is capable of holding the specified amount of plaintext data, - * regardless of the protection mechanism used. - * - * It is possible to save RAM by setting a smaller outward buffer, while keeping - * the default inward 16384 byte buffer to conform to the TLS specification. - * - * The minimum required outward buffer size is determined by the handshake - * protocol's usage. Handshaking will fail if the outward buffer is too small. - * The specific size requirement depends on the configured ciphers and any - * certificate data which is sent during the handshake. - * - * Uncomment to set the maximum plaintext size of the outgoing I/O buffer. - */ -//#define MBEDTLS_SSL_OUT_CONTENT_LEN 16384 - -/** \def MBEDTLS_SSL_DTLS_MAX_BUFFERING - * - * Maximum number of heap-allocated bytes for the purpose of - * DTLS handshake message reassembly and future message buffering. - * - * This should be at least 9/8 * MBEDTLS_SSL_IN_CONTENT_LEN - * to account for a reassembled handshake message of maximum size, - * together with its reassembly bitmap. - * - * A value of 2 * MBEDTLS_SSL_IN_CONTENT_LEN (32768 by default) - * should be sufficient for all practical situations as it allows - * to reassembly a large handshake message (such as a certificate) - * while buffering multiple smaller handshake messages. - * - */ -//#define MBEDTLS_SSL_DTLS_MAX_BUFFERING 32768 - -//#define MBEDTLS_PSK_MAX_LEN 32 /**< Max size of TLS pre-shared keys, in bytes (default 256 or 384 bits) */ -//#define MBEDTLS_SSL_COOKIE_TIMEOUT 60 /**< Default expiration delay of DTLS cookies, in seconds if HAVE_TIME, or in number of cookies issued */ - -/** - * Complete list of ciphersuites to use, in order of preference. - * - * \warning No dependency checking is done on that field! This option can only - * be used to restrict the set of available ciphersuites. It is your - * responsibility to make sure the needed modules are active. - * - * Use this to save a few hundred bytes of ROM (default ordering of all - * available ciphersuites) and a few to a few hundred bytes of RAM. - * - * The value below is only an example, not the default. - */ -//#define MBEDTLS_SSL_CIPHERSUITES MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - -/** - * \def MBEDTLS_SSL_MAX_EARLY_DATA_SIZE - * - * The default maximum amount of 0-RTT data. See the documentation of - * \c mbedtls_ssl_conf_max_early_data_size() for more information. - * - * It must be positive and smaller than UINT32_MAX. - * - * If MBEDTLS_SSL_EARLY_DATA is not defined, this default value does not - * have any impact on the build. - */ -//#define MBEDTLS_SSL_MAX_EARLY_DATA_SIZE 1024 - -/** - * \def MBEDTLS_SSL_TLS1_3_TICKET_AGE_TOLERANCE - * - * Maximum allowed ticket age difference in milliseconds tolerated between - * server and client. Default value is 6000. This is not used in TLS 1.2. - * - * - The client ticket age is the time difference between the time when the - * client proposes to the server to use the ticket and the time the client - * received the ticket from the server. - * - The server ticket age is the time difference between the time when the - * server receives a proposition from the client to use the ticket and the - * time when the ticket was created by the server. - * - * The ages might be different due to the client and server clocks not running - * at the same pace. The typical accuracy of an RTC crystal is ±100 to ±20 parts - * per million (360 to 72 milliseconds per hour). Default tolerance window is - * 6s, thus in the worst case clients and servers must sync up their system time - * every 6000/360/2~=8 hours. - * - * See section 8.3 of the TLS 1.3 specification(RFC 8446) for more information. - */ -//#define MBEDTLS_SSL_TLS1_3_TICKET_AGE_TOLERANCE 6000 - -/** - * \def MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH - * - * Size in bytes of a ticket nonce. This is not used in TLS 1.2. - * - * This must be less than 256. - */ -//#define MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH 32 - -/** - * \def MBEDTLS_SSL_TLS1_3_DEFAULT_NEW_SESSION_TICKETS - * - * Default number of NewSessionTicket messages to be sent by a TLS 1.3 server - * after handshake completion. This is not used in TLS 1.2 and relevant only if - * the MBEDTLS_SSL_SESSION_TICKETS option is enabled. - * - */ -//#define MBEDTLS_SSL_TLS1_3_DEFAULT_NEW_SESSION_TICKETS 1 - -/* X509 options */ -//#define MBEDTLS_X509_MAX_INTERMEDIATE_CA 8 /**< Maximum number of intermediate CAs in a verification chain. */ -//#define MBEDTLS_X509_MAX_FILE_PATH_LEN 512 /**< Maximum length of a path/filename string in bytes including the null terminator character ('\0'). */ - -/** \} name SECTION: Module configuration options */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/md.h b/android/mbedtls/arm64-v8a/include/mbedtls/md.h deleted file mode 100644 index 478e9f766..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/md.h +++ /dev/null @@ -1,526 +0,0 @@ -/** - * \file md.h - * - * \brief This file contains the generic functions for message-digest - * (hashing) and HMAC. - * - * \author Adriaan de Jong - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_MD_H -#define MBEDTLS_MD_H -#include "mbedtls/private_access.h" - -#include - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -/** The selected feature is not available. */ -#define MBEDTLS_ERR_MD_FEATURE_UNAVAILABLE -0x5080 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_MD_BAD_INPUT_DATA -0x5100 -/** Failed to allocate memory. */ -#define MBEDTLS_ERR_MD_ALLOC_FAILED -0x5180 -/** Opening or reading of file failed. */ -#define MBEDTLS_ERR_MD_FILE_IO_ERROR -0x5200 - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Supported message digests. - * - * \warning MD5 and SHA-1 are considered weak message digests and - * their use constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -/* Note: these are aligned with the definitions of PSA_ALG_ macros for hashes, - * in order to enable an efficient implementation of conversion functions. - * This is tested by md_to_from_psa() in test_suite_md. */ -typedef enum { - MBEDTLS_MD_NONE=0, /**< None. */ - MBEDTLS_MD_MD5=0x03, /**< The MD5 message digest. */ - MBEDTLS_MD_RIPEMD160=0x04, /**< The RIPEMD-160 message digest. */ - MBEDTLS_MD_SHA1=0x05, /**< The SHA-1 message digest. */ - MBEDTLS_MD_SHA224=0x08, /**< The SHA-224 message digest. */ - MBEDTLS_MD_SHA256=0x09, /**< The SHA-256 message digest. */ - MBEDTLS_MD_SHA384=0x0a, /**< The SHA-384 message digest. */ - MBEDTLS_MD_SHA512=0x0b, /**< The SHA-512 message digest. */ - MBEDTLS_MD_SHA3_224=0x10, /**< The SHA3-224 message digest. */ - MBEDTLS_MD_SHA3_256=0x11, /**< The SHA3-256 message digest. */ - MBEDTLS_MD_SHA3_384=0x12, /**< The SHA3-384 message digest. */ - MBEDTLS_MD_SHA3_512=0x13, /**< The SHA3-512 message digest. */ -} mbedtls_md_type_t; - -/* Note: this should always be >= PSA_HASH_MAX_SIZE - * in all builds with both CRYPTO_C and MD_LIGHT. - * - * This is to make things easier for modules such as TLS that may define a - * buffer size using MD_MAX_SIZE in a part of the code that's common to PSA - * and legacy, then assume the buffer's size is PSA_HASH_MAX_SIZE in another - * part of the code based on PSA. - */ -#if defined(MBEDTLS_MD_CAN_SHA512) || defined(MBEDTLS_MD_CAN_SHA3_512) -#define MBEDTLS_MD_MAX_SIZE 64 /* longest known is SHA512 */ -#elif defined(MBEDTLS_MD_CAN_SHA384) || defined(MBEDTLS_MD_CAN_SHA3_384) -#define MBEDTLS_MD_MAX_SIZE 48 /* longest known is SHA384 */ -#elif defined(MBEDTLS_MD_CAN_SHA256) || defined(MBEDTLS_MD_CAN_SHA3_256) -#define MBEDTLS_MD_MAX_SIZE 32 /* longest known is SHA256 */ -#elif defined(MBEDTLS_MD_CAN_SHA224) || defined(MBEDTLS_MD_CAN_SHA3_224) -#define MBEDTLS_MD_MAX_SIZE 28 /* longest known is SHA224 */ -#else -#define MBEDTLS_MD_MAX_SIZE 20 /* longest known is SHA1 or RIPE MD-160 - or smaller (MD5 and earlier) */ -#endif - -#if defined(MBEDTLS_MD_CAN_SHA3_224) -#define MBEDTLS_MD_MAX_BLOCK_SIZE 144 /* the longest known is SHA3-224 */ -#elif defined(MBEDTLS_MD_CAN_SHA3_256) -#define MBEDTLS_MD_MAX_BLOCK_SIZE 136 -#elif defined(MBEDTLS_MD_CAN_SHA512) || defined(MBEDTLS_MD_CAN_SHA384) -#define MBEDTLS_MD_MAX_BLOCK_SIZE 128 -#elif defined(MBEDTLS_MD_CAN_SHA3_384) -#define MBEDTLS_MD_MAX_BLOCK_SIZE 104 -#elif defined(MBEDTLS_MD_CAN_SHA3_512) -#define MBEDTLS_MD_MAX_BLOCK_SIZE 72 -#else -#define MBEDTLS_MD_MAX_BLOCK_SIZE 64 -#endif - -/** - * Opaque struct. - * - * Constructed using either #mbedtls_md_info_from_string or - * #mbedtls_md_info_from_type. - * - * Fields can be accessed with #mbedtls_md_get_size, - * #mbedtls_md_get_type and #mbedtls_md_get_name. - */ -/* Defined internally in library/md_wrap.h. */ -typedef struct mbedtls_md_info_t mbedtls_md_info_t; - -/** - * Used internally to indicate whether a context uses legacy or PSA. - * - * Internal use only. - */ -typedef enum { - MBEDTLS_MD_ENGINE_LEGACY = 0, - MBEDTLS_MD_ENGINE_PSA, -} mbedtls_md_engine_t; - -/** - * The generic message-digest context. - */ -typedef struct mbedtls_md_context_t { - /** Information about the associated message digest. */ - const mbedtls_md_info_t *MBEDTLS_PRIVATE(md_info); - -#if defined(MBEDTLS_MD_SOME_PSA) - /** Are hash operations dispatched to PSA or legacy? */ - mbedtls_md_engine_t MBEDTLS_PRIVATE(engine); -#endif - - /** The digest-specific context (legacy) or the PSA operation. */ - void *MBEDTLS_PRIVATE(md_ctx); - -#if defined(MBEDTLS_MD_C) - /** The HMAC part of the context. */ - void *MBEDTLS_PRIVATE(hmac_ctx); -#endif -} mbedtls_md_context_t; - -/** - * \brief This function returns the message-digest information - * associated with the given digest type. - * - * \param md_type The type of digest to search for. - * - * \return The message-digest information associated with \p md_type. - * \return NULL if the associated message-digest information is not found. - */ -const mbedtls_md_info_t *mbedtls_md_info_from_type(mbedtls_md_type_t md_type); - -/** - * \brief This function initializes a message-digest context without - * binding it to a particular message-digest algorithm. - * - * This function should always be called first. It prepares the - * context for mbedtls_md_setup() for binding it to a - * message-digest algorithm. - */ -void mbedtls_md_init(mbedtls_md_context_t *ctx); - -/** - * \brief This function clears the internal structure of \p ctx and - * frees any embedded internal structure, but does not free - * \p ctx itself. - * - * If you have called mbedtls_md_setup() on \p ctx, you must - * call mbedtls_md_free() when you are no longer using the - * context. - * Calling this function if you have previously - * called mbedtls_md_init() and nothing else is optional. - * You must not call this function if you have not called - * mbedtls_md_init(). - */ -void mbedtls_md_free(mbedtls_md_context_t *ctx); - - -/** - * \brief This function selects the message digest algorithm to use, - * and allocates internal structures. - * - * It should be called after mbedtls_md_init() or - * mbedtls_md_free(). Makes it necessary to call - * mbedtls_md_free() later. - * - * \param ctx The context to set up. - * \param md_info The information structure of the message-digest algorithm - * to use. - * \param hmac Defines if HMAC is used. 0: HMAC is not used (saves some memory), - * or non-zero: HMAC is used with this context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - * \return #MBEDTLS_ERR_MD_ALLOC_FAILED on memory-allocation failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_setup(mbedtls_md_context_t *ctx, const mbedtls_md_info_t *md_info, int hmac); - -/** - * \brief This function clones the state of a message-digest - * context. - * - * \note You must call mbedtls_md_setup() on \c dst before calling - * this function. - * - * \note The two contexts must have the same type, - * for example, both are SHA-256. - * - * \warning This function clones the message-digest state, not the - * HMAC state. - * - * \param dst The destination context. - * \param src The context to be cloned. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification failure. - * \return #MBEDTLS_ERR_MD_FEATURE_UNAVAILABLE if both contexts are - * not using the same engine. This can be avoided by moving - * the call to psa_crypto_init() before the first call to - * mbedtls_md_setup(). - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_clone(mbedtls_md_context_t *dst, - const mbedtls_md_context_t *src); - -/** - * \brief This function extracts the message-digest size from the - * message-digest information structure. - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * - * \return The size of the message-digest output in Bytes. - */ -unsigned char mbedtls_md_get_size(const mbedtls_md_info_t *md_info); - -/** - * \brief This function gives the message-digest size associated to - * message-digest type. - * - * \param md_type The message-digest type. - * - * \return The size of the message-digest output in Bytes, - * or 0 if the message-digest type is not known. - */ -static inline unsigned char mbedtls_md_get_size_from_type(mbedtls_md_type_t md_type) -{ - return mbedtls_md_get_size(mbedtls_md_info_from_type(md_type)); -} - -/** - * \brief This function extracts the message-digest type from the - * message-digest information structure. - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * - * \return The type of the message digest. - */ -mbedtls_md_type_t mbedtls_md_get_type(const mbedtls_md_info_t *md_info); - -/** - * \brief This function starts a message-digest computation. - * - * You must call this function after setting up the context - * with mbedtls_md_setup(), and before passing data with - * mbedtls_md_update(). - * - * \param ctx The generic message-digest context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_starts(mbedtls_md_context_t *ctx); - -/** - * \brief This function feeds an input buffer into an ongoing - * message-digest computation. - * - * You must call mbedtls_md_starts() before calling this - * function. You may call this function multiple times. - * Afterwards, call mbedtls_md_finish(). - * - * \param ctx The generic message-digest context. - * \param input The buffer holding the input data. - * \param ilen The length of the input data. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_update(mbedtls_md_context_t *ctx, const unsigned char *input, size_t ilen); - -/** - * \brief This function finishes the digest operation, - * and writes the result to the output buffer. - * - * Call this function after a call to mbedtls_md_starts(), - * followed by any number of calls to mbedtls_md_update(). - * Afterwards, you may either clear the context with - * mbedtls_md_free(), or call mbedtls_md_starts() to reuse - * the context for another digest operation with the same - * algorithm. - * - * \param ctx The generic message-digest context. - * \param output The buffer for the generic message-digest checksum result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_finish(mbedtls_md_context_t *ctx, unsigned char *output); - -/** - * \brief This function calculates the message-digest of a buffer, - * with respect to a configurable message-digest algorithm - * in a single call. - * - * The result is calculated as - * Output = message_digest(input buffer). - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * \param input The buffer holding the data. - * \param ilen The length of the input data. - * \param output The generic message-digest checksum result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md(const mbedtls_md_info_t *md_info, const unsigned char *input, size_t ilen, - unsigned char *output); - -/** - * \brief This function returns the list of digests supported by the - * generic digest module. - * - * \note The list starts with the strongest available hashes. - * - * \return A statically allocated array of digests. Each element - * in the returned list is an integer belonging to the - * message-digest enumeration #mbedtls_md_type_t. - * The last entry is 0. - */ -const int *mbedtls_md_list(void); - -/** - * \brief This function returns the message-digest information - * associated with the given digest name. - * - * \param md_name The name of the digest to search for. - * - * \return The message-digest information associated with \p md_name. - * \return NULL if the associated message-digest information is not found. - */ -const mbedtls_md_info_t *mbedtls_md_info_from_string(const char *md_name); - -/** - * \brief This function returns the name of the message digest for - * the message-digest information structure given. - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * - * \return The name of the message digest. - */ -const char *mbedtls_md_get_name(const mbedtls_md_info_t *md_info); - -/** - * \brief This function returns the message-digest information - * from the given context. - * - * \param ctx The context from which to extract the information. - * This must be initialized (or \c NULL). - * - * \return The message-digest information associated with \p ctx. - * \return \c NULL if \p ctx is \c NULL. - */ -const mbedtls_md_info_t *mbedtls_md_info_from_ctx( - const mbedtls_md_context_t *ctx); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief This function calculates the message-digest checksum - * result of the contents of the provided file. - * - * The result is calculated as - * Output = message_digest(file contents). - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * \param path The input file name. - * \param output The generic message-digest checksum result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_FILE_IO_ERROR on an I/O error accessing - * the file pointed by \p path. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA if \p md_info was NULL. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_file(const mbedtls_md_info_t *md_info, const char *path, - unsigned char *output); -#endif /* MBEDTLS_FS_IO */ - -/** - * \brief This function sets the HMAC key and prepares to - * authenticate a new message. - * - * Call this function after mbedtls_md_setup(), to use - * the MD context for an HMAC calculation, then call - * mbedtls_md_hmac_update() to provide the input data, and - * mbedtls_md_hmac_finish() to get the HMAC value. - * - * \param ctx The message digest context containing an embedded HMAC - * context. - * \param key The HMAC secret key. - * \param keylen The length of the HMAC key in Bytes. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_hmac_starts(mbedtls_md_context_t *ctx, const unsigned char *key, - size_t keylen); - -/** - * \brief This function feeds an input buffer into an ongoing HMAC - * computation. - * - * Call mbedtls_md_hmac_starts() or mbedtls_md_hmac_reset() - * before calling this function. - * You may call this function multiple times to pass the - * input piecewise. - * Afterwards, call mbedtls_md_hmac_finish(). - * - * \param ctx The message digest context containing an embedded HMAC - * context. - * \param input The buffer holding the input data. - * \param ilen The length of the input data. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_hmac_update(mbedtls_md_context_t *ctx, const unsigned char *input, - size_t ilen); - -/** - * \brief This function finishes the HMAC operation, and writes - * the result to the output buffer. - * - * Call this function after mbedtls_md_hmac_starts() and - * mbedtls_md_hmac_update() to get the HMAC value. Afterwards - * you may either call mbedtls_md_free() to clear the context, - * or call mbedtls_md_hmac_reset() to reuse the context with - * the same HMAC key. - * - * \param ctx The message digest context containing an embedded HMAC - * context. - * \param output The generic HMAC checksum result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_hmac_finish(mbedtls_md_context_t *ctx, unsigned char *output); - -/** - * \brief This function prepares to authenticate a new message with - * the same key as the previous HMAC operation. - * - * You may call this function after mbedtls_md_hmac_finish(). - * Afterwards call mbedtls_md_hmac_update() to pass the new - * input. - * - * \param ctx The message digest context containing an embedded HMAC - * context. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_hmac_reset(mbedtls_md_context_t *ctx); - -/** - * \brief This function calculates the full generic HMAC - * on the input buffer with the provided key. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The HMAC result is calculated as - * output = generic HMAC(hmac key, input buffer). - * - * \param md_info The information structure of the message-digest algorithm - * to use. - * \param key The HMAC secret key. - * \param keylen The length of the HMAC secret key in Bytes. - * \param input The buffer holding the input data. - * \param ilen The length of the input data. - * \param output The generic HMAC result. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MD_BAD_INPUT_DATA on parameter-verification - * failure. - */ -MBEDTLS_CHECK_RETURN_TYPICAL -int mbedtls_md_hmac(const mbedtls_md_info_t *md_info, const unsigned char *key, size_t keylen, - const unsigned char *input, size_t ilen, - unsigned char *output); - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_MD_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/md5.h b/android/mbedtls/arm64-v8a/include/mbedtls/md5.h deleted file mode 100644 index 6bf0754a4..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/md5.h +++ /dev/null @@ -1,190 +0,0 @@ -/** - * \file md5.h - * - * \brief MD5 message digest algorithm (hash function) - * - * \warning MD5 is considered a weak message digest and its use constitutes a - * security risk. We recommend considering stronger message - * digests instead. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_MD5_H -#define MBEDTLS_MD5_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_MD5_ALT) -// Regular implementation -// - -/** - * \brief MD5 context structure - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -typedef struct mbedtls_md5_context { - uint32_t MBEDTLS_PRIVATE(total)[2]; /*!< number of bytes processed */ - uint32_t MBEDTLS_PRIVATE(state)[4]; /*!< intermediate digest state */ - unsigned char MBEDTLS_PRIVATE(buffer)[64]; /*!< data block being processed */ -} -mbedtls_md5_context; - -#else /* MBEDTLS_MD5_ALT */ -#include "md5_alt.h" -#endif /* MBEDTLS_MD5_ALT */ - -/** - * \brief Initialize MD5 context - * - * \param ctx MD5 context to be initialized - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -void mbedtls_md5_init(mbedtls_md5_context *ctx); - -/** - * \brief Clear MD5 context - * - * \param ctx MD5 context to be cleared - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -void mbedtls_md5_free(mbedtls_md5_context *ctx); - -/** - * \brief Clone (the state of) an MD5 context - * - * \param dst The destination context - * \param src The context to be cloned - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -void mbedtls_md5_clone(mbedtls_md5_context *dst, - const mbedtls_md5_context *src); - -/** - * \brief MD5 context setup - * - * \param ctx context to be initialized - * - * \return 0 if successful - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_md5_starts(mbedtls_md5_context *ctx); - -/** - * \brief MD5 process buffer - * - * \param ctx MD5 context - * \param input buffer holding the data - * \param ilen length of the input data - * - * \return 0 if successful - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_md5_update(mbedtls_md5_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief MD5 final digest - * - * \param ctx MD5 context - * \param output MD5 checksum result - * - * \return 0 if successful - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_md5_finish(mbedtls_md5_context *ctx, - unsigned char output[16]); - -/** - * \brief MD5 process data block (internal use only) - * - * \param ctx MD5 context - * \param data buffer holding one block of data - * - * \return 0 if successful - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_internal_md5_process(mbedtls_md5_context *ctx, - const unsigned char data[64]); - -/** - * \brief Output = MD5( input buffer ) - * - * \param input buffer holding the data - * \param ilen length of the input data - * \param output MD5 checksum result - * - * \return 0 if successful - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_md5(const unsigned char *input, - size_t ilen, - unsigned char output[16]); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - * - * \warning MD5 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -int mbedtls_md5_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_md5.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/memory_buffer_alloc.h b/android/mbedtls/arm64-v8a/include/mbedtls/memory_buffer_alloc.h deleted file mode 100644 index b527d9b66..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/memory_buffer_alloc.h +++ /dev/null @@ -1,142 +0,0 @@ -/** - * \file memory_buffer_alloc.h - * - * \brief Buffer-based memory allocator - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_MEMORY_BUFFER_ALLOC_H -#define MBEDTLS_MEMORY_BUFFER_ALLOC_H - -#include "mbedtls/build_info.h" - -#include - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_MEMORY_ALIGN_MULTIPLE) -#define MBEDTLS_MEMORY_ALIGN_MULTIPLE 4 /**< Align on multiples of this value */ -#endif - -/** \} name SECTION: Module settings */ - -#define MBEDTLS_MEMORY_VERIFY_NONE 0 -#define MBEDTLS_MEMORY_VERIFY_ALLOC (1 << 0) -#define MBEDTLS_MEMORY_VERIFY_FREE (1 << 1) -#define MBEDTLS_MEMORY_VERIFY_ALWAYS (MBEDTLS_MEMORY_VERIFY_ALLOC | \ - MBEDTLS_MEMORY_VERIFY_FREE) - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Initialize use of stack-based memory allocator. - * The stack-based allocator does memory management inside the - * presented buffer and does not call calloc() and free(). - * It sets the global mbedtls_calloc() and mbedtls_free() pointers - * to its own functions. - * (Provided mbedtls_calloc() and mbedtls_free() are thread-safe if - * MBEDTLS_THREADING_C is defined) - * - * \note This code is not optimized and provides a straight-forward - * implementation of a stack-based memory allocator. - * - * \param buf buffer to use as heap - * \param len size of the buffer - */ -void mbedtls_memory_buffer_alloc_init(unsigned char *buf, size_t len); - -/** - * \brief Free the mutex for thread-safety and clear remaining memory - */ -void mbedtls_memory_buffer_alloc_free(void); - -/** - * \brief Determine when the allocator should automatically verify the state - * of the entire chain of headers / meta-data. - * (Default: MBEDTLS_MEMORY_VERIFY_NONE) - * - * \param verify One of MBEDTLS_MEMORY_VERIFY_NONE, MBEDTLS_MEMORY_VERIFY_ALLOC, - * MBEDTLS_MEMORY_VERIFY_FREE or MBEDTLS_MEMORY_VERIFY_ALWAYS - */ -void mbedtls_memory_buffer_set_verify(int verify); - -#if defined(MBEDTLS_MEMORY_DEBUG) -/** - * \brief Print out the status of the allocated memory (primarily for use - * after a program should have de-allocated all memory) - * Prints out a list of 'still allocated' blocks and their stack - * trace if MBEDTLS_MEMORY_BACKTRACE is defined. - */ -void mbedtls_memory_buffer_alloc_status(void); - -/** - * \brief Get the number of alloc/free so far. - * - * \param alloc_count Number of allocations. - * \param free_count Number of frees. - */ -void mbedtls_memory_buffer_alloc_count_get(size_t *alloc_count, size_t *free_count); - -/** - * \brief Get the peak heap usage so far - * - * \param max_used Peak number of bytes in use or committed. This - * includes bytes in allocated blocks too small to split - * into smaller blocks but larger than the requested size. - * \param max_blocks Peak number of blocks in use, including free and used - */ -void mbedtls_memory_buffer_alloc_max_get(size_t *max_used, size_t *max_blocks); - -/** - * \brief Reset peak statistics - */ -void mbedtls_memory_buffer_alloc_max_reset(void); - -/** - * \brief Get the current heap usage - * - * \param cur_used Current number of bytes in use or committed. This - * includes bytes in allocated blocks too small to split - * into smaller blocks but larger than the requested size. - * \param cur_blocks Current number of blocks in use, including free and used - */ -void mbedtls_memory_buffer_alloc_cur_get(size_t *cur_used, size_t *cur_blocks); -#endif /* MBEDTLS_MEMORY_DEBUG */ - -/** - * \brief Verifies that all headers in the memory buffer are correct - * and contain sane values. Helps debug buffer-overflow errors. - * - * Prints out first failure if MBEDTLS_MEMORY_DEBUG is defined. - * Prints out full header information if MBEDTLS_MEMORY_DEBUG - * is defined. (Includes stack trace information for each block if - * MBEDTLS_MEMORY_BACKTRACE is defined as well). - * - * \return 0 if verified, 1 otherwise - */ -int mbedtls_memory_buffer_alloc_verify(void); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if a test failed - */ -int mbedtls_memory_buffer_alloc_self_test(int verbose); -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* memory_buffer_alloc.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/net_sockets.h b/android/mbedtls/arm64-v8a/include/mbedtls/net_sockets.h deleted file mode 100644 index 8e69bc0fb..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/net_sockets.h +++ /dev/null @@ -1,299 +0,0 @@ -/** - * \file net_sockets.h - * - * \brief Network sockets abstraction layer to integrate Mbed TLS into a - * BSD-style sockets API. - * - * The network sockets module provides an example integration of the - * Mbed TLS library into a BSD sockets implementation. The module is - * intended to be an example of how Mbed TLS can be integrated into a - * networking stack, as well as to be Mbed TLS's network integration - * for its supported platforms. - * - * The module is intended only to be used with the Mbed TLS library and - * is not intended to be used by third party application software - * directly. - * - * The supported platforms are as follows: - * * Microsoft Windows and Windows CE - * * POSIX/Unix platforms including Linux, OS X - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_NET_SOCKETS_H -#define MBEDTLS_NET_SOCKETS_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/ssl.h" - -#include -#include - -/** Failed to open a socket. */ -#define MBEDTLS_ERR_NET_SOCKET_FAILED -0x0042 -/** The connection to the given server / port failed. */ -#define MBEDTLS_ERR_NET_CONNECT_FAILED -0x0044 -/** Binding of the socket failed. */ -#define MBEDTLS_ERR_NET_BIND_FAILED -0x0046 -/** Could not listen on the socket. */ -#define MBEDTLS_ERR_NET_LISTEN_FAILED -0x0048 -/** Could not accept the incoming connection. */ -#define MBEDTLS_ERR_NET_ACCEPT_FAILED -0x004A -/** Reading information from the socket failed. */ -#define MBEDTLS_ERR_NET_RECV_FAILED -0x004C -/** Sending information through the socket failed. */ -#define MBEDTLS_ERR_NET_SEND_FAILED -0x004E -/** Connection was reset by peer. */ -#define MBEDTLS_ERR_NET_CONN_RESET -0x0050 -/** Failed to get an IP address for the given hostname. */ -#define MBEDTLS_ERR_NET_UNKNOWN_HOST -0x0052 -/** Buffer is too small to hold the data. */ -#define MBEDTLS_ERR_NET_BUFFER_TOO_SMALL -0x0043 -/** The context is invalid, eg because it was free()ed. */ -#define MBEDTLS_ERR_NET_INVALID_CONTEXT -0x0045 -/** Polling the net context failed. */ -#define MBEDTLS_ERR_NET_POLL_FAILED -0x0047 -/** Input invalid. */ -#define MBEDTLS_ERR_NET_BAD_INPUT_DATA -0x0049 - -#define MBEDTLS_NET_LISTEN_BACKLOG 10 /**< The backlog that listen() should use. */ - -#define MBEDTLS_NET_PROTO_TCP 0 /**< The TCP transport protocol */ -#define MBEDTLS_NET_PROTO_UDP 1 /**< The UDP transport protocol */ - -#define MBEDTLS_NET_POLL_READ 1 /**< Used in \c mbedtls_net_poll to check for pending data */ -#define MBEDTLS_NET_POLL_WRITE 2 /**< Used in \c mbedtls_net_poll to check if write possible */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Wrapper type for sockets. - * - * Currently backed by just a file descriptor, but might be more in the future - * (eg two file descriptors for combined IPv4 + IPv6 support, or additional - * structures for hand-made UDP demultiplexing). - */ -typedef struct mbedtls_net_context { - /** The underlying file descriptor. - * - * This field is only guaranteed to be present on POSIX/Unix-like platforms. - * On other platforms, it may have a different type, have a different - * meaning, or be absent altogether. - */ - int fd; -} -mbedtls_net_context; - -/** - * \brief Initialize a context - * Just makes the context ready to be used or freed safely. - * - * \param ctx Context to initialize - */ -void mbedtls_net_init(mbedtls_net_context *ctx); - -/** - * \brief Initiate a connection with host:port in the given protocol - * - * \param ctx Socket to use - * \param host Host to connect to - * \param port Port to connect to - * \param proto Protocol: MBEDTLS_NET_PROTO_TCP or MBEDTLS_NET_PROTO_UDP - * - * \return 0 if successful, or one of: - * MBEDTLS_ERR_NET_SOCKET_FAILED, - * MBEDTLS_ERR_NET_UNKNOWN_HOST, - * MBEDTLS_ERR_NET_CONNECT_FAILED - * - * \note Sets the socket in connected mode even with UDP. - */ -int mbedtls_net_connect(mbedtls_net_context *ctx, const char *host, const char *port, int proto); - -/** - * \brief Create a receiving socket on bind_ip:port in the chosen - * protocol. If bind_ip == NULL, all interfaces are bound. - * - * \param ctx Socket to use - * \param bind_ip IP to bind to, can be NULL - * \param port Port number to use - * \param proto Protocol: MBEDTLS_NET_PROTO_TCP or MBEDTLS_NET_PROTO_UDP - * - * \return 0 if successful, or one of: - * MBEDTLS_ERR_NET_SOCKET_FAILED, - * MBEDTLS_ERR_NET_UNKNOWN_HOST, - * MBEDTLS_ERR_NET_BIND_FAILED, - * MBEDTLS_ERR_NET_LISTEN_FAILED - * - * \note Regardless of the protocol, opens the sockets and binds it. - * In addition, make the socket listening if protocol is TCP. - */ -int mbedtls_net_bind(mbedtls_net_context *ctx, const char *bind_ip, const char *port, int proto); - -/** - * \brief Accept a connection from a remote client - * - * \param bind_ctx Relevant socket - * \param client_ctx Will contain the connected client socket - * \param client_ip Will contain the client IP address, can be NULL - * \param buf_size Size of the client_ip buffer - * \param cip_len Will receive the size of the client IP written, - * can be NULL if client_ip is null - * - * \return 0 if successful, or - * MBEDTLS_ERR_NET_SOCKET_FAILED, - * MBEDTLS_ERR_NET_BIND_FAILED, - * MBEDTLS_ERR_NET_ACCEPT_FAILED, or - * MBEDTLS_ERR_NET_BUFFER_TOO_SMALL if buf_size is too small, - * MBEDTLS_ERR_SSL_WANT_READ if bind_fd was set to - * non-blocking and accept() would block. - */ -int mbedtls_net_accept(mbedtls_net_context *bind_ctx, - mbedtls_net_context *client_ctx, - void *client_ip, size_t buf_size, size_t *cip_len); - -/** - * \brief Check and wait for the context to be ready for read/write - * - * \note The current implementation of this function uses - * select() and returns an error if the file descriptor - * is \c FD_SETSIZE or greater. - * - * \param ctx Socket to check - * \param rw Bitflag composed of MBEDTLS_NET_POLL_READ and - * MBEDTLS_NET_POLL_WRITE specifying the events - * to wait for: - * - If MBEDTLS_NET_POLL_READ is set, the function - * will return as soon as the net context is available - * for reading. - * - If MBEDTLS_NET_POLL_WRITE is set, the function - * will return as soon as the net context is available - * for writing. - * \param timeout Maximal amount of time to wait before returning, - * in milliseconds. If \c timeout is zero, the - * function returns immediately. If \c timeout is - * -1u, the function blocks potentially indefinitely. - * - * \return Bitmask composed of MBEDTLS_NET_POLL_READ/WRITE - * on success or timeout, or a negative return code otherwise. - */ -int mbedtls_net_poll(mbedtls_net_context *ctx, uint32_t rw, uint32_t timeout); - -/** - * \brief Set the socket blocking - * - * \param ctx Socket to set - * - * \return 0 if successful, or a non-zero error code - */ -int mbedtls_net_set_block(mbedtls_net_context *ctx); - -/** - * \brief Set the socket non-blocking - * - * \param ctx Socket to set - * - * \return 0 if successful, or a non-zero error code - */ -int mbedtls_net_set_nonblock(mbedtls_net_context *ctx); - -/** - * \brief Portable usleep helper - * - * \param usec Amount of microseconds to sleep - * - * \note Real amount of time slept will not be less than - * select()'s timeout granularity (typically, 10ms). - */ -void mbedtls_net_usleep(unsigned long usec); - -/** - * \brief Read at most 'len' characters. If no error occurs, - * the actual amount read is returned. - * - * \param ctx Socket - * \param buf The buffer to write to - * \param len Maximum length of the buffer - * - * \return the number of bytes received, - * or a non-zero error code; with a non-blocking socket, - * MBEDTLS_ERR_SSL_WANT_READ indicates read() would block. - */ -int mbedtls_net_recv(void *ctx, unsigned char *buf, size_t len); - -/** - * \brief Write at most 'len' characters. If no error occurs, - * the actual amount written is returned. - * - * \param ctx Socket - * \param buf The buffer to read from - * \param len The length of the buffer - * - * \return the number of bytes sent, - * or a non-zero error code; with a non-blocking socket, - * MBEDTLS_ERR_SSL_WANT_WRITE indicates write() would block. - */ -int mbedtls_net_send(void *ctx, const unsigned char *buf, size_t len); - -/** - * \brief Read at most 'len' characters, blocking for at most - * 'timeout' seconds. If no error occurs, the actual amount - * read is returned. - * - * \note The current implementation of this function uses - * select() and returns an error if the file descriptor - * is \c FD_SETSIZE or greater. - * - * \param ctx Socket - * \param buf The buffer to write to - * \param len Maximum length of the buffer - * \param timeout Maximum number of milliseconds to wait for data - * 0 means no timeout (wait forever) - * - * \return The number of bytes received if successful. - * MBEDTLS_ERR_SSL_TIMEOUT if the operation timed out. - * MBEDTLS_ERR_SSL_WANT_READ if interrupted by a signal. - * Another negative error code (MBEDTLS_ERR_NET_xxx) - * for other failures. - * - * \note This function will block (until data becomes available or - * timeout is reached) even if the socket is set to - * non-blocking. Handling timeouts with non-blocking reads - * requires a different strategy. - */ -int mbedtls_net_recv_timeout(void *ctx, unsigned char *buf, size_t len, - uint32_t timeout); - -/** - * \brief Closes down the connection and free associated data - * - * \param ctx The context to close - * - * \note This function frees and clears data associated with the - * context but does not free the memory pointed to by \p ctx. - * This memory is the responsibility of the caller. - */ -void mbedtls_net_close(mbedtls_net_context *ctx); - -/** - * \brief Gracefully shutdown the connection and free associated data - * - * \param ctx The context to free - * - * \note This function frees and clears data associated with the - * context but does not free the memory pointed to by \p ctx. - * This memory is the responsibility of the caller. - */ -void mbedtls_net_free(mbedtls_net_context *ctx); - -#ifdef __cplusplus -} -#endif - -#endif /* net_sockets.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/nist_kw.h b/android/mbedtls/arm64-v8a/include/mbedtls/nist_kw.h deleted file mode 100644 index d353f3d1a..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/nist_kw.h +++ /dev/null @@ -1,166 +0,0 @@ -/** - * \file nist_kw.h - * - * \brief This file provides an API for key wrapping (KW) and key wrapping with - * padding (KWP) as defined in NIST SP 800-38F. - * https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-38F.pdf - * - * Key wrapping specifies a deterministic authenticated-encryption mode - * of operation, according to NIST SP 800-38F: Recommendation for - * Block Cipher Modes of Operation: Methods for Key Wrapping. Its - * purpose is to protect cryptographic keys. - * - * Its equivalent is RFC 3394 for KW, and RFC 5649 for KWP. - * https://tools.ietf.org/html/rfc3394 - * https://tools.ietf.org/html/rfc5649 - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_NIST_KW_H -#define MBEDTLS_NIST_KW_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/cipher.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef enum { - MBEDTLS_KW_MODE_KW = 0, - MBEDTLS_KW_MODE_KWP = 1 -} mbedtls_nist_kw_mode_t; - -#if !defined(MBEDTLS_NIST_KW_ALT) -// Regular implementation -// - -/** - * \brief The key wrapping context-type definition. The key wrapping context is passed - * to the APIs called. - * - * \note The definition of this type may change in future library versions. - * Don't make any assumptions on this context! - */ -typedef struct { - mbedtls_cipher_context_t MBEDTLS_PRIVATE(cipher_ctx); /*!< The cipher context used. */ -} mbedtls_nist_kw_context; - -#else /* MBEDTLS_NIST_key wrapping_ALT */ -#include "nist_kw_alt.h" -#endif /* MBEDTLS_NIST_KW_ALT */ - -/** - * \brief This function initializes the specified key wrapping context - * to make references valid and prepare the context - * for mbedtls_nist_kw_setkey() or mbedtls_nist_kw_free(). - * - * \param ctx The key wrapping context to initialize. - * - */ -void mbedtls_nist_kw_init(mbedtls_nist_kw_context *ctx); - -/** - * \brief This function initializes the key wrapping context set in the - * \p ctx parameter and sets the encryption key. - * - * \param ctx The key wrapping context. - * \param cipher The 128-bit block cipher to use. Only AES is supported. - * \param key The Key Encryption Key (KEK). - * \param keybits The KEK size in bits. This must be acceptable by the cipher. - * \param is_wrap Specify whether the operation within the context is wrapping or unwrapping - * - * \return \c 0 on success. - * \return \c MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA for any invalid input. - * \return \c MBEDTLS_ERR_CIPHER_FEATURE_UNAVAILABLE for 128-bit block ciphers - * which are not supported. - * \return cipher-specific error code on failure of the underlying cipher. - */ -int mbedtls_nist_kw_setkey(mbedtls_nist_kw_context *ctx, - mbedtls_cipher_id_t cipher, - const unsigned char *key, - unsigned int keybits, - const int is_wrap); - -/** - * \brief This function releases and clears the specified key wrapping context - * and underlying cipher sub-context. - * - * \param ctx The key wrapping context to clear. - */ -void mbedtls_nist_kw_free(mbedtls_nist_kw_context *ctx); - -/** - * \brief This function encrypts a buffer using key wrapping. - * - * \param ctx The key wrapping context to use for encryption. - * \param mode The key wrapping mode to use (MBEDTLS_KW_MODE_KW or MBEDTLS_KW_MODE_KWP) - * \param input The buffer holding the input data. - * \param in_len The length of the input data in Bytes. - * The input uses units of 8 Bytes called semiblocks. - *
  • For KW mode: a multiple of 8 bytes between 16 and 2^57-8 inclusive.
    • - *
  • For KWP mode: any length between 1 and 2^32-1 inclusive.
- * \param[out] output The buffer holding the output data. - *
  • For KW mode: Must be at least 8 bytes larger than \p in_len.
    • - *
  • For KWP mode: Must be at least 8 bytes larger rounded up to a multiple of - * 8 bytes for KWP (15 bytes at most).
- * \param[out] out_len The number of bytes written to the output buffer. \c 0 on failure. - * \param[in] out_size The capacity of the output buffer. - * - * \return \c 0 on success. - * \return \c MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA for invalid input length. - * \return cipher-specific error code on failure of the underlying cipher. - */ -int mbedtls_nist_kw_wrap(mbedtls_nist_kw_context *ctx, mbedtls_nist_kw_mode_t mode, - const unsigned char *input, size_t in_len, - unsigned char *output, size_t *out_len, size_t out_size); - -/** - * \brief This function decrypts a buffer using key wrapping. - * - * \param ctx The key wrapping context to use for decryption. - * \param mode The key wrapping mode to use (MBEDTLS_KW_MODE_KW or MBEDTLS_KW_MODE_KWP) - * \param input The buffer holding the input data. - * \param in_len The length of the input data in Bytes. - * The input uses units of 8 Bytes called semiblocks. - * The input must be a multiple of semiblocks. - *
  • For KW mode: a multiple of 8 bytes between 24 and 2^57 inclusive.
    • - *
  • For KWP mode: a multiple of 8 bytes between 16 and 2^32 inclusive.
- * \param[out] output The buffer holding the output data. - * The output buffer's minimal length is 8 bytes shorter than \p in_len. - * \param[out] out_len The number of bytes written to the output buffer. \c 0 on failure. - * For KWP mode, the length could be up to 15 bytes shorter than \p in_len, - * depending on how much padding was added to the data. - * \param[in] out_size The capacity of the output buffer. - * - * \return \c 0 on success. - * \return \c MBEDTLS_ERR_CIPHER_BAD_INPUT_DATA for invalid input length. - * \return \c MBEDTLS_ERR_CIPHER_AUTH_FAILED for verification failure of the ciphertext. - * \return cipher-specific error code on failure of the underlying cipher. - */ -int mbedtls_nist_kw_unwrap(mbedtls_nist_kw_context *ctx, mbedtls_nist_kw_mode_t mode, - const unsigned char *input, size_t in_len, - unsigned char *output, size_t *out_len, size_t out_size); - - -#if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C) -/** - * \brief The key wrapping checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_nist_kw_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST && MBEDTLS_AES_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_NIST_KW_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/oid.h b/android/mbedtls/arm64-v8a/include/mbedtls/oid.h deleted file mode 100644 index fdc25ebf8..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/oid.h +++ /dev/null @@ -1,727 +0,0 @@ -/** - * \file oid.h - * - * \brief Object Identifier (OID) database - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_OID_H -#define MBEDTLS_OID_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/asn1.h" -#include "mbedtls/pk.h" - -#include - -#if defined(MBEDTLS_CIPHER_C) -#include "mbedtls/cipher.h" -#endif - -#include "mbedtls/md.h" - -/** OID is not found. */ -#define MBEDTLS_ERR_OID_NOT_FOUND -0x002E -/** output buffer is too small */ -#define MBEDTLS_ERR_OID_BUF_TOO_SMALL -0x000B - -/* This is for the benefit of X.509, but defined here in order to avoid - * having a "backwards" include of x.509.h here */ -/* - * X.509 extension types (internal, arbitrary values for bitsets) - */ -#define MBEDTLS_OID_X509_EXT_AUTHORITY_KEY_IDENTIFIER (1 << 0) -#define MBEDTLS_OID_X509_EXT_SUBJECT_KEY_IDENTIFIER (1 << 1) -#define MBEDTLS_OID_X509_EXT_KEY_USAGE (1 << 2) -#define MBEDTLS_OID_X509_EXT_CERTIFICATE_POLICIES (1 << 3) -#define MBEDTLS_OID_X509_EXT_POLICY_MAPPINGS (1 << 4) -#define MBEDTLS_OID_X509_EXT_SUBJECT_ALT_NAME (1 << 5) -#define MBEDTLS_OID_X509_EXT_ISSUER_ALT_NAME (1 << 6) -#define MBEDTLS_OID_X509_EXT_SUBJECT_DIRECTORY_ATTRS (1 << 7) -#define MBEDTLS_OID_X509_EXT_BASIC_CONSTRAINTS (1 << 8) -#define MBEDTLS_OID_X509_EXT_NAME_CONSTRAINTS (1 << 9) -#define MBEDTLS_OID_X509_EXT_POLICY_CONSTRAINTS (1 << 10) -#define MBEDTLS_OID_X509_EXT_EXTENDED_KEY_USAGE (1 << 11) -#define MBEDTLS_OID_X509_EXT_CRL_DISTRIBUTION_POINTS (1 << 12) -#define MBEDTLS_OID_X509_EXT_INIHIBIT_ANYPOLICY (1 << 13) -#define MBEDTLS_OID_X509_EXT_FRESHEST_CRL (1 << 14) -#define MBEDTLS_OID_X509_EXT_NS_CERT_TYPE (1 << 16) - -/* - * Maximum number of OID components allowed - */ -#define MBEDTLS_OID_MAX_COMPONENTS 128 - -/* - * Top level OID tuples - */ -#define MBEDTLS_OID_ISO_MEMBER_BODIES "\x2a" /* {iso(1) member-body(2)} */ -#define MBEDTLS_OID_ISO_IDENTIFIED_ORG "\x2b" /* {iso(1) identified-organization(3)} */ -#define MBEDTLS_OID_ISO_CCITT_DS "\x55" /* {joint-iso-ccitt(2) ds(5)} */ -#define MBEDTLS_OID_ISO_ITU_COUNTRY "\x60" /* {joint-iso-itu-t(2) country(16)} */ - -/* - * ISO Member bodies OID parts - */ -#define MBEDTLS_OID_COUNTRY_US "\x86\x48" /* {us(840)} */ -#define MBEDTLS_OID_ORG_RSA_DATA_SECURITY "\x86\xf7\x0d" /* {rsadsi(113549)} */ -#define MBEDTLS_OID_RSA_COMPANY MBEDTLS_OID_ISO_MEMBER_BODIES MBEDTLS_OID_COUNTRY_US \ - MBEDTLS_OID_ORG_RSA_DATA_SECURITY /* {iso(1) member-body(2) us(840) rsadsi(113549)} */ -#define MBEDTLS_OID_ORG_ANSI_X9_62 "\xce\x3d" /* ansi-X9-62(10045) */ -#define MBEDTLS_OID_ANSI_X9_62 MBEDTLS_OID_ISO_MEMBER_BODIES MBEDTLS_OID_COUNTRY_US \ - MBEDTLS_OID_ORG_ANSI_X9_62 - -/* - * ISO Identified organization OID parts - */ -#define MBEDTLS_OID_ORG_DOD "\x06" /* {dod(6)} */ -#define MBEDTLS_OID_ORG_OIW "\x0e" -#define MBEDTLS_OID_OIW_SECSIG MBEDTLS_OID_ORG_OIW "\x03" -#define MBEDTLS_OID_OIW_SECSIG_ALG MBEDTLS_OID_OIW_SECSIG "\x02" -#define MBEDTLS_OID_OIW_SECSIG_SHA1 MBEDTLS_OID_OIW_SECSIG_ALG "\x1a" -#define MBEDTLS_OID_ORG_THAWTE "\x65" /* thawte(101) */ -#define MBEDTLS_OID_THAWTE MBEDTLS_OID_ISO_IDENTIFIED_ORG \ - MBEDTLS_OID_ORG_THAWTE -#define MBEDTLS_OID_ORG_CERTICOM "\x81\x04" /* certicom(132) */ -#define MBEDTLS_OID_CERTICOM MBEDTLS_OID_ISO_IDENTIFIED_ORG \ - MBEDTLS_OID_ORG_CERTICOM -#define MBEDTLS_OID_ORG_TELETRUST "\x24" /* teletrust(36) */ -#define MBEDTLS_OID_TELETRUST MBEDTLS_OID_ISO_IDENTIFIED_ORG \ - MBEDTLS_OID_ORG_TELETRUST - -/* - * ISO ITU OID parts - */ -#define MBEDTLS_OID_ORGANIZATION "\x01" /* {organization(1)} */ -#define MBEDTLS_OID_ISO_ITU_US_ORG MBEDTLS_OID_ISO_ITU_COUNTRY MBEDTLS_OID_COUNTRY_US \ - MBEDTLS_OID_ORGANIZATION /* {joint-iso-itu-t(2) country(16) us(840) organization(1)} */ - -#define MBEDTLS_OID_ORG_GOV "\x65" /* {gov(101)} */ -#define MBEDTLS_OID_GOV MBEDTLS_OID_ISO_ITU_US_ORG MBEDTLS_OID_ORG_GOV /* {joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101)} */ - -#define MBEDTLS_OID_ORG_NETSCAPE "\x86\xF8\x42" /* {netscape(113730)} */ -#define MBEDTLS_OID_NETSCAPE MBEDTLS_OID_ISO_ITU_US_ORG MBEDTLS_OID_ORG_NETSCAPE /* Netscape OID {joint-iso-itu-t(2) country(16) us(840) organization(1) netscape(113730)} */ - -/* ISO arc for standard certificate and CRL extensions */ -#define MBEDTLS_OID_ID_CE MBEDTLS_OID_ISO_CCITT_DS "\x1D" /**< id-ce OBJECT IDENTIFIER ::= {joint-iso-ccitt(2) ds(5) 29} */ - -#define MBEDTLS_OID_NIST_ALG MBEDTLS_OID_GOV "\x03\x04" /** { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithm(4) */ - -/** - * Private Internet Extensions - * { iso(1) identified-organization(3) dod(6) internet(1) - * security(5) mechanisms(5) pkix(7) } - */ -#define MBEDTLS_OID_INTERNET MBEDTLS_OID_ISO_IDENTIFIED_ORG MBEDTLS_OID_ORG_DOD \ - "\x01" -#define MBEDTLS_OID_PKIX MBEDTLS_OID_INTERNET "\x05\x05\x07" - -/* - * Arc for standard naming attributes - */ -#define MBEDTLS_OID_AT MBEDTLS_OID_ISO_CCITT_DS "\x04" /**< id-at OBJECT IDENTIFIER ::= {joint-iso-ccitt(2) ds(5) 4} */ -#define MBEDTLS_OID_AT_CN MBEDTLS_OID_AT "\x03" /**< id-at-commonName AttributeType:= {id-at 3} */ -#define MBEDTLS_OID_AT_SUR_NAME MBEDTLS_OID_AT "\x04" /**< id-at-surName AttributeType:= {id-at 4} */ -#define MBEDTLS_OID_AT_SERIAL_NUMBER MBEDTLS_OID_AT "\x05" /**< id-at-serialNumber AttributeType:= {id-at 5} */ -#define MBEDTLS_OID_AT_COUNTRY MBEDTLS_OID_AT "\x06" /**< id-at-countryName AttributeType:= {id-at 6} */ -#define MBEDTLS_OID_AT_LOCALITY MBEDTLS_OID_AT "\x07" /**< id-at-locality AttributeType:= {id-at 7} */ -#define MBEDTLS_OID_AT_STATE MBEDTLS_OID_AT "\x08" /**< id-at-state AttributeType:= {id-at 8} */ -#define MBEDTLS_OID_AT_ORGANIZATION MBEDTLS_OID_AT "\x0A" /**< id-at-organizationName AttributeType:= {id-at 10} */ -#define MBEDTLS_OID_AT_ORG_UNIT MBEDTLS_OID_AT "\x0B" /**< id-at-organizationalUnitName AttributeType:= {id-at 11} */ -#define MBEDTLS_OID_AT_TITLE MBEDTLS_OID_AT "\x0C" /**< id-at-title AttributeType:= {id-at 12} */ -#define MBEDTLS_OID_AT_POSTAL_ADDRESS MBEDTLS_OID_AT "\x10" /**< id-at-postalAddress AttributeType:= {id-at 16} */ -#define MBEDTLS_OID_AT_POSTAL_CODE MBEDTLS_OID_AT "\x11" /**< id-at-postalCode AttributeType:= {id-at 17} */ -#define MBEDTLS_OID_AT_GIVEN_NAME MBEDTLS_OID_AT "\x2A" /**< id-at-givenName AttributeType:= {id-at 42} */ -#define MBEDTLS_OID_AT_INITIALS MBEDTLS_OID_AT "\x2B" /**< id-at-initials AttributeType:= {id-at 43} */ -#define MBEDTLS_OID_AT_GENERATION_QUALIFIER MBEDTLS_OID_AT "\x2C" /**< id-at-generationQualifier AttributeType:= {id-at 44} */ -#define MBEDTLS_OID_AT_UNIQUE_IDENTIFIER MBEDTLS_OID_AT "\x2D" /**< id-at-uniqueIdentifier AttributeType:= {id-at 45} */ -#define MBEDTLS_OID_AT_DN_QUALIFIER MBEDTLS_OID_AT "\x2E" /**< id-at-dnQualifier AttributeType:= {id-at 46} */ -#define MBEDTLS_OID_AT_PSEUDONYM MBEDTLS_OID_AT "\x41" /**< id-at-pseudonym AttributeType:= {id-at 65} */ - -#define MBEDTLS_OID_UID "\x09\x92\x26\x89\x93\xF2\x2C\x64\x01\x01" /** id-domainComponent AttributeType:= {itu-t(0) data(9) pss(2342) ucl(19200300) pilot(100) pilotAttributeType(1) uid(1)} */ -#define MBEDTLS_OID_DOMAIN_COMPONENT "\x09\x92\x26\x89\x93\xF2\x2C\x64\x01\x19" /** id-domainComponent AttributeType:= {itu-t(0) data(9) pss(2342) ucl(19200300) pilot(100) pilotAttributeType(1) domainComponent(25)} */ - -/* - * OIDs for standard certificate extensions - */ -#define MBEDTLS_OID_AUTHORITY_KEY_IDENTIFIER MBEDTLS_OID_ID_CE "\x23" /**< id-ce-authorityKeyIdentifier OBJECT IDENTIFIER ::= { id-ce 35 } */ -#define MBEDTLS_OID_SUBJECT_KEY_IDENTIFIER MBEDTLS_OID_ID_CE "\x0E" /**< id-ce-subjectKeyIdentifier OBJECT IDENTIFIER ::= { id-ce 14 } */ -#define MBEDTLS_OID_KEY_USAGE MBEDTLS_OID_ID_CE "\x0F" /**< id-ce-keyUsage OBJECT IDENTIFIER ::= { id-ce 15 } */ -#define MBEDTLS_OID_CERTIFICATE_POLICIES MBEDTLS_OID_ID_CE "\x20" /**< id-ce-certificatePolicies OBJECT IDENTIFIER ::= { id-ce 32 } */ -#define MBEDTLS_OID_POLICY_MAPPINGS MBEDTLS_OID_ID_CE "\x21" /**< id-ce-policyMappings OBJECT IDENTIFIER ::= { id-ce 33 } */ -#define MBEDTLS_OID_SUBJECT_ALT_NAME MBEDTLS_OID_ID_CE "\x11" /**< id-ce-subjectAltName OBJECT IDENTIFIER ::= { id-ce 17 } */ -#define MBEDTLS_OID_ISSUER_ALT_NAME MBEDTLS_OID_ID_CE "\x12" /**< id-ce-issuerAltName OBJECT IDENTIFIER ::= { id-ce 18 } */ -#define MBEDTLS_OID_SUBJECT_DIRECTORY_ATTRS MBEDTLS_OID_ID_CE "\x09" /**< id-ce-subjectDirectoryAttributes OBJECT IDENTIFIER ::= { id-ce 9 } */ -#define MBEDTLS_OID_BASIC_CONSTRAINTS MBEDTLS_OID_ID_CE "\x13" /**< id-ce-basicConstraints OBJECT IDENTIFIER ::= { id-ce 19 } */ -#define MBEDTLS_OID_NAME_CONSTRAINTS MBEDTLS_OID_ID_CE "\x1E" /**< id-ce-nameConstraints OBJECT IDENTIFIER ::= { id-ce 30 } */ -#define MBEDTLS_OID_POLICY_CONSTRAINTS MBEDTLS_OID_ID_CE "\x24" /**< id-ce-policyConstraints OBJECT IDENTIFIER ::= { id-ce 36 } */ -#define MBEDTLS_OID_EXTENDED_KEY_USAGE MBEDTLS_OID_ID_CE "\x25" /**< id-ce-extKeyUsage OBJECT IDENTIFIER ::= { id-ce 37 } */ -#define MBEDTLS_OID_CRL_DISTRIBUTION_POINTS MBEDTLS_OID_ID_CE "\x1F" /**< id-ce-cRLDistributionPoints OBJECT IDENTIFIER ::= { id-ce 31 } */ -#define MBEDTLS_OID_INIHIBIT_ANYPOLICY MBEDTLS_OID_ID_CE "\x36" /**< id-ce-inhibitAnyPolicy OBJECT IDENTIFIER ::= { id-ce 54 } */ -#define MBEDTLS_OID_FRESHEST_CRL MBEDTLS_OID_ID_CE "\x2E" /**< id-ce-freshestCRL OBJECT IDENTIFIER ::= { id-ce 46 } */ - -/* - * Certificate policies - */ -#define MBEDTLS_OID_ANY_POLICY MBEDTLS_OID_CERTIFICATE_POLICIES "\x00" /**< anyPolicy OBJECT IDENTIFIER ::= { id-ce-certificatePolicies 0 } */ - -/* - * Netscape certificate extensions - */ -#define MBEDTLS_OID_NS_CERT MBEDTLS_OID_NETSCAPE "\x01" -#define MBEDTLS_OID_NS_CERT_TYPE MBEDTLS_OID_NS_CERT "\x01" -#define MBEDTLS_OID_NS_BASE_URL MBEDTLS_OID_NS_CERT "\x02" -#define MBEDTLS_OID_NS_REVOCATION_URL MBEDTLS_OID_NS_CERT "\x03" -#define MBEDTLS_OID_NS_CA_REVOCATION_URL MBEDTLS_OID_NS_CERT "\x04" -#define MBEDTLS_OID_NS_RENEWAL_URL MBEDTLS_OID_NS_CERT "\x07" -#define MBEDTLS_OID_NS_CA_POLICY_URL MBEDTLS_OID_NS_CERT "\x08" -#define MBEDTLS_OID_NS_SSL_SERVER_NAME MBEDTLS_OID_NS_CERT "\x0C" -#define MBEDTLS_OID_NS_COMMENT MBEDTLS_OID_NS_CERT "\x0D" -#define MBEDTLS_OID_NS_DATA_TYPE MBEDTLS_OID_NETSCAPE "\x02" -#define MBEDTLS_OID_NS_CERT_SEQUENCE MBEDTLS_OID_NS_DATA_TYPE "\x05" - -/* - * OIDs for CRL extensions - */ -#define MBEDTLS_OID_PRIVATE_KEY_USAGE_PERIOD MBEDTLS_OID_ID_CE "\x10" -#define MBEDTLS_OID_CRL_NUMBER MBEDTLS_OID_ID_CE "\x14" /**< id-ce-cRLNumber OBJECT IDENTIFIER ::= { id-ce 20 } */ - -/* - * X.509 v3 Extended key usage OIDs - */ -#define MBEDTLS_OID_ANY_EXTENDED_KEY_USAGE MBEDTLS_OID_EXTENDED_KEY_USAGE "\x00" /**< anyExtendedKeyUsage OBJECT IDENTIFIER ::= { id-ce-extKeyUsage 0 } */ - -#define MBEDTLS_OID_KP MBEDTLS_OID_PKIX "\x03" /**< id-kp OBJECT IDENTIFIER ::= { id-pkix 3 } */ -#define MBEDTLS_OID_SERVER_AUTH MBEDTLS_OID_KP "\x01" /**< id-kp-serverAuth OBJECT IDENTIFIER ::= { id-kp 1 } */ -#define MBEDTLS_OID_CLIENT_AUTH MBEDTLS_OID_KP "\x02" /**< id-kp-clientAuth OBJECT IDENTIFIER ::= { id-kp 2 } */ -#define MBEDTLS_OID_CODE_SIGNING MBEDTLS_OID_KP "\x03" /**< id-kp-codeSigning OBJECT IDENTIFIER ::= { id-kp 3 } */ -#define MBEDTLS_OID_EMAIL_PROTECTION MBEDTLS_OID_KP "\x04" /**< id-kp-emailProtection OBJECT IDENTIFIER ::= { id-kp 4 } */ -#define MBEDTLS_OID_TIME_STAMPING MBEDTLS_OID_KP "\x08" /**< id-kp-timeStamping OBJECT IDENTIFIER ::= { id-kp 8 } */ -#define MBEDTLS_OID_OCSP_SIGNING MBEDTLS_OID_KP "\x09" /**< id-kp-OCSPSigning OBJECT IDENTIFIER ::= { id-kp 9 } */ - -/** - * Wi-SUN Alliance Field Area Network - * { iso(1) identified-organization(3) dod(6) internet(1) - * private(4) enterprise(1) WiSUN(45605) FieldAreaNetwork(1) } - */ -#define MBEDTLS_OID_WISUN_FAN MBEDTLS_OID_INTERNET "\x04\x01\x82\xe4\x25\x01" - -#define MBEDTLS_OID_ON MBEDTLS_OID_PKIX "\x08" /**< id-on OBJECT IDENTIFIER ::= { id-pkix 8 } */ -#define MBEDTLS_OID_ON_HW_MODULE_NAME MBEDTLS_OID_ON "\x04" /**< id-on-hardwareModuleName OBJECT IDENTIFIER ::= { id-on 4 } */ - -/* - * PKCS definition OIDs - */ - -#define MBEDTLS_OID_PKCS MBEDTLS_OID_RSA_COMPANY "\x01" /**< pkcs OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) 1 } */ -#define MBEDTLS_OID_PKCS1 MBEDTLS_OID_PKCS "\x01" /**< pkcs-1 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 1 } */ -#define MBEDTLS_OID_PKCS5 MBEDTLS_OID_PKCS "\x05" /**< pkcs-5 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 5 } */ -#define MBEDTLS_OID_PKCS7 MBEDTLS_OID_PKCS "\x07" /**< pkcs-7 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 7 } */ -#define MBEDTLS_OID_PKCS9 MBEDTLS_OID_PKCS "\x09" /**< pkcs-9 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 9 } */ -#define MBEDTLS_OID_PKCS12 MBEDTLS_OID_PKCS "\x0c" /**< pkcs-12 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) pkcs(1) 12 } */ - -/* - * PKCS#1 OIDs - */ -#define MBEDTLS_OID_PKCS1_RSA MBEDTLS_OID_PKCS1 "\x01" /**< rsaEncryption OBJECT IDENTIFIER ::= { pkcs-1 1 } */ -#define MBEDTLS_OID_PKCS1_MD5 MBEDTLS_OID_PKCS1 "\x04" /**< md5WithRSAEncryption ::= { pkcs-1 4 } */ -#define MBEDTLS_OID_PKCS1_SHA1 MBEDTLS_OID_PKCS1 "\x05" /**< sha1WithRSAEncryption ::= { pkcs-1 5 } */ -#define MBEDTLS_OID_PKCS1_SHA224 MBEDTLS_OID_PKCS1 "\x0e" /**< sha224WithRSAEncryption ::= { pkcs-1 14 } */ -#define MBEDTLS_OID_PKCS1_SHA256 MBEDTLS_OID_PKCS1 "\x0b" /**< sha256WithRSAEncryption ::= { pkcs-1 11 } */ -#define MBEDTLS_OID_PKCS1_SHA384 MBEDTLS_OID_PKCS1 "\x0c" /**< sha384WithRSAEncryption ::= { pkcs-1 12 } */ -#define MBEDTLS_OID_PKCS1_SHA512 MBEDTLS_OID_PKCS1 "\x0d" /**< sha512WithRSAEncryption ::= { pkcs-1 13 } */ - -#define MBEDTLS_OID_RSA_SHA_OBS "\x2B\x0E\x03\x02\x1D" - -#define MBEDTLS_OID_PKCS9_EMAIL MBEDTLS_OID_PKCS9 "\x01" /**< emailAddress AttributeType ::= { pkcs-9 1 } */ - -/* RFC 4055 */ -#define MBEDTLS_OID_RSASSA_PSS MBEDTLS_OID_PKCS1 "\x0a" /**< id-RSASSA-PSS ::= { pkcs-1 10 } */ -#define MBEDTLS_OID_MGF1 MBEDTLS_OID_PKCS1 "\x08" /**< id-mgf1 ::= { pkcs-1 8 } */ - -/* - * Digest algorithms - */ -#define MBEDTLS_OID_DIGEST_ALG_MD5 MBEDTLS_OID_RSA_COMPANY "\x02\x05" /**< id-mbedtls_md5 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 5 } */ -#define MBEDTLS_OID_DIGEST_ALG_SHA1 MBEDTLS_OID_ISO_IDENTIFIED_ORG \ - MBEDTLS_OID_OIW_SECSIG_SHA1 /**< id-mbedtls_sha1 OBJECT IDENTIFIER ::= { iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) 26 } */ -#define MBEDTLS_OID_DIGEST_ALG_SHA224 MBEDTLS_OID_NIST_ALG "\x02\x04" /**< id-sha224 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistalgorithm(4) hashalgs(2) 4 } */ -#define MBEDTLS_OID_DIGEST_ALG_SHA256 MBEDTLS_OID_NIST_ALG "\x02\x01" /**< id-mbedtls_sha256 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistalgorithm(4) hashalgs(2) 1 } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA384 MBEDTLS_OID_NIST_ALG "\x02\x02" /**< id-sha384 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistalgorithm(4) hashalgs(2) 2 } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA512 MBEDTLS_OID_NIST_ALG "\x02\x03" /**< id-mbedtls_sha512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistalgorithm(4) hashalgs(2) 3 } */ - -#define MBEDTLS_OID_DIGEST_ALG_RIPEMD160 MBEDTLS_OID_TELETRUST "\x03\x02\x01" /**< id-ripemd160 OBJECT IDENTIFIER :: { iso(1) identified-organization(3) teletrust(36) algorithm(3) hashAlgorithm(2) ripemd160(1) } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA3_224 MBEDTLS_OID_NIST_ALG "\x02\x07" /**< id-sha3-224 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) sha3-224(7) } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA3_256 MBEDTLS_OID_NIST_ALG "\x02\x08" /**< id-sha3-256 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) sha3-256(8) } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA3_384 MBEDTLS_OID_NIST_ALG "\x02\x09" /**< id-sha3-384 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) sha3-384(9) } */ - -#define MBEDTLS_OID_DIGEST_ALG_SHA3_512 MBEDTLS_OID_NIST_ALG "\x02\x0a" /**< id-sha3-512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) sha3-512(10) } */ - - -#define MBEDTLS_OID_HMAC_SHA1 MBEDTLS_OID_RSA_COMPANY "\x02\x07" /**< id-hmacWithSHA1 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 7 } */ - -#define MBEDTLS_OID_HMAC_SHA224 MBEDTLS_OID_RSA_COMPANY "\x02\x08" /**< id-hmacWithSHA224 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 8 } */ - -#define MBEDTLS_OID_HMAC_SHA256 MBEDTLS_OID_RSA_COMPANY "\x02\x09" /**< id-hmacWithSHA256 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 9 } */ - -#define MBEDTLS_OID_HMAC_SHA384 MBEDTLS_OID_RSA_COMPANY "\x02\x0A" /**< id-hmacWithSHA384 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 10 } */ - -#define MBEDTLS_OID_HMAC_SHA512 MBEDTLS_OID_RSA_COMPANY "\x02\x0B" /**< id-hmacWithSHA512 OBJECT IDENTIFIER ::= { iso(1) member-body(2) us(840) rsadsi(113549) digestAlgorithm(2) 11 } */ - -#define MBEDTLS_OID_HMAC_SHA3_224 MBEDTLS_OID_NIST_ALG "\x02\x0d" /**< id-hmacWithSHA3-512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) hmacWithSHA3-224(13) } */ - -#define MBEDTLS_OID_HMAC_SHA3_256 MBEDTLS_OID_NIST_ALG "\x02\x0e" /**< id-hmacWithSHA3-512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) hmacWithSHA3-256(14) } */ - -#define MBEDTLS_OID_HMAC_SHA3_384 MBEDTLS_OID_NIST_ALG "\x02\x0f" /**< id-hmacWithSHA3-512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) hmacWithSHA3-384(15) } */ - -#define MBEDTLS_OID_HMAC_SHA3_512 MBEDTLS_OID_NIST_ALG "\x02\x10" /**< id-hmacWithSHA3-512 OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) hashalgs(2) hmacWithSHA3-512(16) } */ - -#define MBEDTLS_OID_HMAC_RIPEMD160 MBEDTLS_OID_INTERNET "\x05\x05\x08\x01\x04" /**< id-hmacWithSHA1 OBJECT IDENTIFIER ::= {iso(1) iso-identified-organization(3) dod(6) internet(1) security(5) mechanisms(5) ipsec(8) isakmpOakley(1) hmacRIPEMD160(4)} */ - -/* - * Encryption algorithms, - * the following standardized object identifiers are specified at - * https://datatracker.ietf.org/doc/html/rfc8018#appendix-C. - */ -#define MBEDTLS_OID_DES_CBC MBEDTLS_OID_ISO_IDENTIFIED_ORG \ - MBEDTLS_OID_OIW_SECSIG_ALG "\x07" /**< desCBC OBJECT IDENTIFIER ::= { iso(1) identified-organization(3) oiw(14) secsig(3) algorithms(2) 7 } */ -#define MBEDTLS_OID_DES_EDE3_CBC MBEDTLS_OID_RSA_COMPANY "\x03\x07" /**< des-ede3-cbc OBJECT IDENTIFIER ::= { iso(1) member-body(2) -- us(840) rsadsi(113549) encryptionAlgorithm(3) 7 } */ -#define MBEDTLS_OID_AES MBEDTLS_OID_NIST_ALG "\x01" /** aes OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithm(4) 1 } */ -#define MBEDTLS_OID_AES_128_CBC MBEDTLS_OID_AES "\x02" /** aes128-cbc-pad OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) aes(1) aes128-CBC-PAD(2) } */ -#define MBEDTLS_OID_AES_192_CBC MBEDTLS_OID_AES "\x16" /** aes192-cbc-pad OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) aes(1) aes192-CBC-PAD(22) } */ -#define MBEDTLS_OID_AES_256_CBC MBEDTLS_OID_AES "\x2a" /** aes256-cbc-pad OBJECT IDENTIFIER ::= { joint-iso-itu-t(2) country(16) us(840) organization(1) gov(101) csor(3) nistAlgorithms(4) aes(1) aes256-CBC-PAD(42) } */ - -/* - * Key Wrapping algorithms - */ -/* - * RFC 5649 - */ -#define MBEDTLS_OID_AES128_KW MBEDTLS_OID_AES "\x05" /** id-aes128-wrap OBJECT IDENTIFIER ::= { aes 5 } */ -#define MBEDTLS_OID_AES128_KWP MBEDTLS_OID_AES "\x08" /** id-aes128-wrap-pad OBJECT IDENTIFIER ::= { aes 8 } */ -#define MBEDTLS_OID_AES192_KW MBEDTLS_OID_AES "\x19" /** id-aes192-wrap OBJECT IDENTIFIER ::= { aes 25 } */ -#define MBEDTLS_OID_AES192_KWP MBEDTLS_OID_AES "\x1c" /** id-aes192-wrap-pad OBJECT IDENTIFIER ::= { aes 28 } */ -#define MBEDTLS_OID_AES256_KW MBEDTLS_OID_AES "\x2d" /** id-aes256-wrap OBJECT IDENTIFIER ::= { aes 45 } */ -#define MBEDTLS_OID_AES256_KWP MBEDTLS_OID_AES "\x30" /** id-aes256-wrap-pad OBJECT IDENTIFIER ::= { aes 48 } */ -/* - * PKCS#5 OIDs - */ -#define MBEDTLS_OID_PKCS5_PBKDF2 MBEDTLS_OID_PKCS5 "\x0c" /**< id-PBKDF2 OBJECT IDENTIFIER ::= {pkcs-5 12} */ -#define MBEDTLS_OID_PKCS5_PBES2 MBEDTLS_OID_PKCS5 "\x0d" /**< id-PBES2 OBJECT IDENTIFIER ::= {pkcs-5 13} */ -#define MBEDTLS_OID_PKCS5_PBMAC1 MBEDTLS_OID_PKCS5 "\x0e" /**< id-PBMAC1 OBJECT IDENTIFIER ::= {pkcs-5 14} */ - -/* - * PKCS#5 PBES1 algorithms - */ -#define MBEDTLS_OID_PKCS5_PBE_MD5_DES_CBC MBEDTLS_OID_PKCS5 "\x03" /**< pbeWithMD5AndDES-CBC OBJECT IDENTIFIER ::= {pkcs-5 3} */ -#define MBEDTLS_OID_PKCS5_PBE_MD5_RC2_CBC MBEDTLS_OID_PKCS5 "\x06" /**< pbeWithMD5AndRC2-CBC OBJECT IDENTIFIER ::= {pkcs-5 6} */ -#define MBEDTLS_OID_PKCS5_PBE_SHA1_DES_CBC MBEDTLS_OID_PKCS5 "\x0a" /**< pbeWithSHA1AndDES-CBC OBJECT IDENTIFIER ::= {pkcs-5 10} */ -#define MBEDTLS_OID_PKCS5_PBE_SHA1_RC2_CBC MBEDTLS_OID_PKCS5 "\x0b" /**< pbeWithSHA1AndRC2-CBC OBJECT IDENTIFIER ::= {pkcs-5 11} */ - -/* - * PKCS#7 OIDs - */ -#define MBEDTLS_OID_PKCS7_DATA MBEDTLS_OID_PKCS7 "\x01" /**< Content type is Data OBJECT IDENTIFIER ::= {pkcs-7 1} */ -#define MBEDTLS_OID_PKCS7_SIGNED_DATA MBEDTLS_OID_PKCS7 "\x02" /**< Content type is Signed Data OBJECT IDENTIFIER ::= {pkcs-7 2} */ -#define MBEDTLS_OID_PKCS7_ENVELOPED_DATA MBEDTLS_OID_PKCS7 "\x03" /**< Content type is Enveloped Data OBJECT IDENTIFIER ::= {pkcs-7 3} */ -#define MBEDTLS_OID_PKCS7_SIGNED_AND_ENVELOPED_DATA MBEDTLS_OID_PKCS7 "\x04" /**< Content type is Signed and Enveloped Data OBJECT IDENTIFIER ::= {pkcs-7 4} */ -#define MBEDTLS_OID_PKCS7_DIGESTED_DATA MBEDTLS_OID_PKCS7 "\x05" /**< Content type is Digested Data OBJECT IDENTIFIER ::= {pkcs-7 5} */ -#define MBEDTLS_OID_PKCS7_ENCRYPTED_DATA MBEDTLS_OID_PKCS7 "\x06" /**< Content type is Encrypted Data OBJECT IDENTIFIER ::= {pkcs-7 6} */ - -/* - * PKCS#8 OIDs - */ -#define MBEDTLS_OID_PKCS9_CSR_EXT_REQ MBEDTLS_OID_PKCS9 "\x0e" /**< extensionRequest OBJECT IDENTIFIER ::= {pkcs-9 14} */ - -/* - * PKCS#12 PBE OIDs - */ -#define MBEDTLS_OID_PKCS12_PBE MBEDTLS_OID_PKCS12 "\x01" /**< pkcs-12PbeIds OBJECT IDENTIFIER ::= {pkcs-12 1} */ - -#define MBEDTLS_OID_PKCS12_PBE_SHA1_DES3_EDE_CBC MBEDTLS_OID_PKCS12_PBE "\x03" /**< pbeWithSHAAnd3-KeyTripleDES-CBC OBJECT IDENTIFIER ::= {pkcs-12PbeIds 3} */ -#define MBEDTLS_OID_PKCS12_PBE_SHA1_DES2_EDE_CBC MBEDTLS_OID_PKCS12_PBE "\x04" /**< pbeWithSHAAnd2-KeyTripleDES-CBC OBJECT IDENTIFIER ::= {pkcs-12PbeIds 4} */ -#define MBEDTLS_OID_PKCS12_PBE_SHA1_RC2_128_CBC MBEDTLS_OID_PKCS12_PBE "\x05" /**< pbeWithSHAAnd128BitRC2-CBC OBJECT IDENTIFIER ::= {pkcs-12PbeIds 5} */ -#define MBEDTLS_OID_PKCS12_PBE_SHA1_RC2_40_CBC MBEDTLS_OID_PKCS12_PBE "\x06" /**< pbeWithSHAAnd40BitRC2-CBC OBJECT IDENTIFIER ::= {pkcs-12PbeIds 6} */ - -/* - * EC key algorithms from RFC 5480 - */ - -/* id-ecPublicKey OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) keyType(2) 1 } */ -#define MBEDTLS_OID_EC_ALG_UNRESTRICTED MBEDTLS_OID_ANSI_X9_62 "\x02\01" - -/* id-ecDH OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) - * schemes(1) ecdh(12) } */ -#define MBEDTLS_OID_EC_ALG_ECDH MBEDTLS_OID_CERTICOM "\x01\x0c" - -/* - * ECParameters namedCurve identifiers, from RFC 5480, RFC 5639, and SEC2 - */ - -/* secp192r1 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) curves(3) prime(1) 1 } */ -#define MBEDTLS_OID_EC_GRP_SECP192R1 MBEDTLS_OID_ANSI_X9_62 "\x03\x01\x01" - -/* secp224r1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 33 } */ -#define MBEDTLS_OID_EC_GRP_SECP224R1 MBEDTLS_OID_CERTICOM "\x00\x21" - -/* secp256r1 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) curves(3) prime(1) 7 } */ -#define MBEDTLS_OID_EC_GRP_SECP256R1 MBEDTLS_OID_ANSI_X9_62 "\x03\x01\x07" - -/* secp384r1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 34 } */ -#define MBEDTLS_OID_EC_GRP_SECP384R1 MBEDTLS_OID_CERTICOM "\x00\x22" - -/* secp521r1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 35 } */ -#define MBEDTLS_OID_EC_GRP_SECP521R1 MBEDTLS_OID_CERTICOM "\x00\x23" - -/* secp192k1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 31 } */ -#define MBEDTLS_OID_EC_GRP_SECP192K1 MBEDTLS_OID_CERTICOM "\x00\x1f" - -/* secp224k1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 32 } */ -#define MBEDTLS_OID_EC_GRP_SECP224K1 MBEDTLS_OID_CERTICOM "\x00\x20" - -/* secp256k1 OBJECT IDENTIFIER ::= { - * iso(1) identified-organization(3) certicom(132) curve(0) 10 } */ -#define MBEDTLS_OID_EC_GRP_SECP256K1 MBEDTLS_OID_CERTICOM "\x00\x0a" - -/* RFC 5639 4.1 - * ecStdCurvesAndGeneration OBJECT IDENTIFIER::= {iso(1) - * identified-organization(3) teletrust(36) algorithm(3) signature- - * algorithm(3) ecSign(2) 8} - * ellipticCurve OBJECT IDENTIFIER ::= {ecStdCurvesAndGeneration 1} - * versionOne OBJECT IDENTIFIER ::= {ellipticCurve 1} */ -#define MBEDTLS_OID_EC_BRAINPOOL_V1 MBEDTLS_OID_TELETRUST "\x03\x03\x02\x08\x01\x01" - -/* brainpoolP256r1 OBJECT IDENTIFIER ::= {versionOne 7} */ -#define MBEDTLS_OID_EC_GRP_BP256R1 MBEDTLS_OID_EC_BRAINPOOL_V1 "\x07" - -/* brainpoolP384r1 OBJECT IDENTIFIER ::= {versionOne 11} */ -#define MBEDTLS_OID_EC_GRP_BP384R1 MBEDTLS_OID_EC_BRAINPOOL_V1 "\x0B" - -/* brainpoolP512r1 OBJECT IDENTIFIER ::= {versionOne 13} */ -#define MBEDTLS_OID_EC_GRP_BP512R1 MBEDTLS_OID_EC_BRAINPOOL_V1 "\x0D" - -/* - * SEC1 C.1 - * - * prime-field OBJECT IDENTIFIER ::= { id-fieldType 1 } - * id-fieldType OBJECT IDENTIFIER ::= { ansi-X9-62 fieldType(1)} - */ -#define MBEDTLS_OID_ANSI_X9_62_FIELD_TYPE MBEDTLS_OID_ANSI_X9_62 "\x01" -#define MBEDTLS_OID_ANSI_X9_62_PRIME_FIELD MBEDTLS_OID_ANSI_X9_62_FIELD_TYPE "\x01" - -/* - * ECDSA signature identifiers, from RFC 5480 - */ -#define MBEDTLS_OID_ANSI_X9_62_SIG MBEDTLS_OID_ANSI_X9_62 "\x04" /* signatures(4) */ -#define MBEDTLS_OID_ANSI_X9_62_SIG_SHA2 MBEDTLS_OID_ANSI_X9_62_SIG "\x03" /* ecdsa-with-SHA2(3) */ - -/* ecdsa-with-SHA1 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) signatures(4) 1 } */ -#define MBEDTLS_OID_ECDSA_SHA1 MBEDTLS_OID_ANSI_X9_62_SIG "\x01" - -/* ecdsa-with-SHA224 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) signatures(4) - * ecdsa-with-SHA2(3) 1 } */ -#define MBEDTLS_OID_ECDSA_SHA224 MBEDTLS_OID_ANSI_X9_62_SIG_SHA2 "\x01" - -/* ecdsa-with-SHA256 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) signatures(4) - * ecdsa-with-SHA2(3) 2 } */ -#define MBEDTLS_OID_ECDSA_SHA256 MBEDTLS_OID_ANSI_X9_62_SIG_SHA2 "\x02" - -/* ecdsa-with-SHA384 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) signatures(4) - * ecdsa-with-SHA2(3) 3 } */ -#define MBEDTLS_OID_ECDSA_SHA384 MBEDTLS_OID_ANSI_X9_62_SIG_SHA2 "\x03" - -/* ecdsa-with-SHA512 OBJECT IDENTIFIER ::= { - * iso(1) member-body(2) us(840) ansi-X9-62(10045) signatures(4) - * ecdsa-with-SHA2(3) 4 } */ -#define MBEDTLS_OID_ECDSA_SHA512 MBEDTLS_OID_ANSI_X9_62_SIG_SHA2 "\x04" - -/* - * EC key algorithms from RFC 8410 - */ - -#define MBEDTLS_OID_X25519 MBEDTLS_OID_THAWTE "\x6e" /**< id-X25519 OBJECT IDENTIFIER ::= { 1 3 101 110 } */ -#define MBEDTLS_OID_X448 MBEDTLS_OID_THAWTE "\x6f" /**< id-X448 OBJECT IDENTIFIER ::= { 1 3 101 111 } */ -#define MBEDTLS_OID_ED25519 MBEDTLS_OID_THAWTE "\x70" /**< id-Ed25519 OBJECT IDENTIFIER ::= { 1 3 101 112 } */ -#define MBEDTLS_OID_ED448 MBEDTLS_OID_THAWTE "\x71" /**< id-Ed448 OBJECT IDENTIFIER ::= { 1 3 101 113 } */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Base OID descriptor structure - */ -typedef struct mbedtls_oid_descriptor_t { - const char *MBEDTLS_PRIVATE(asn1); /*!< OID ASN.1 representation */ - size_t MBEDTLS_PRIVATE(asn1_len); /*!< length of asn1 */ -#if !defined(MBEDTLS_X509_REMOVE_INFO) - const char *MBEDTLS_PRIVATE(name); /*!< official name (e.g. from RFC) */ - const char *MBEDTLS_PRIVATE(description); /*!< human friendly description */ -#endif -} mbedtls_oid_descriptor_t; - -/** - * \brief Translate an ASN.1 OID into its numeric representation - * (e.g. "\x2A\x86\x48\x86\xF7\x0D" into "1.2.840.113549") - * - * \param buf buffer to put representation in - * \param size size of the buffer - * \param oid OID to translate - * - * \return Length of the string written (excluding final NULL) or - * MBEDTLS_ERR_OID_BUF_TOO_SMALL in case of error - */ -int mbedtls_oid_get_numeric_string(char *buf, size_t size, const mbedtls_asn1_buf *oid); - -/** - * \brief Translate a string containing a dotted-decimal - * representation of an ASN.1 OID into its encoded form - * (e.g. "1.2.840.113549" into "\x2A\x86\x48\x86\xF7\x0D"). - * On success, this function allocates oid->buf from the - * heap. It must be freed by the caller using mbedtls_free(). - * - * \param oid #mbedtls_asn1_buf to populate with the DER-encoded OID - * \param oid_str string representation of the OID to parse - * \param size length of the OID string, not including any null terminator - * - * \return 0 if successful - * \return #MBEDTLS_ERR_ASN1_INVALID_DATA if \p oid_str does not - * represent a valid OID - * \return #MBEDTLS_ERR_ASN1_ALLOC_FAILED if the function fails to - * allocate oid->buf - */ -int mbedtls_oid_from_numeric_string(mbedtls_asn1_buf *oid, const char *oid_str, size_t size); - -/** - * \brief Translate an X.509 extension OID into local values - * - * \param oid OID to use - * \param ext_type place to store the extension type - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_x509_ext_type(const mbedtls_asn1_buf *oid, int *ext_type); - -/** - * \brief Translate an X.509 attribute type OID into the short name - * (e.g. the OID for an X520 Common Name into "CN") - * - * \param oid OID to use - * \param short_name place to store the string pointer - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_attr_short_name(const mbedtls_asn1_buf *oid, const char **short_name); - -/** - * \brief Translate PublicKeyAlgorithm OID into pk_type - * - * \param oid OID to use - * \param pk_alg place to store public key algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_pk_alg(const mbedtls_asn1_buf *oid, mbedtls_pk_type_t *pk_alg); - -/** - * \brief Translate pk_type into PublicKeyAlgorithm OID - * - * \param pk_alg Public key type to look for - * \param oid place to store ASN.1 OID string pointer - * \param olen length of the OID - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_oid_by_pk_alg(mbedtls_pk_type_t pk_alg, - const char **oid, size_t *olen); - -#if defined(MBEDTLS_PK_HAVE_ECC_KEYS) -/** - * \brief Translate NamedCurve OID into an EC group identifier - * - * \param oid OID to use - * \param grp_id place to store group id - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_ec_grp(const mbedtls_asn1_buf *oid, mbedtls_ecp_group_id *grp_id); - -/** - * \brief Translate EC group identifier into NamedCurve OID - * - * \param grp_id EC group identifier - * \param oid place to store ASN.1 OID string pointer - * \param olen length of the OID - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_oid_by_ec_grp(mbedtls_ecp_group_id grp_id, - const char **oid, size_t *olen); - -/** - * \brief Translate AlgorithmIdentifier OID into an EC group identifier, - * for curves that are directly encoded at this level - * - * \param oid OID to use - * \param grp_id place to store group id - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_ec_grp_algid(const mbedtls_asn1_buf *oid, mbedtls_ecp_group_id *grp_id); - -/** - * \brief Translate EC group identifier into AlgorithmIdentifier OID, - * for curves that are directly encoded at this level - * - * \param grp_id EC group identifier - * \param oid place to store ASN.1 OID string pointer - * \param olen length of the OID - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_oid_by_ec_grp_algid(mbedtls_ecp_group_id grp_id, - const char **oid, size_t *olen); -#endif /* MBEDTLS_PK_HAVE_ECC_KEYS */ - -/** - * \brief Translate SignatureAlgorithm OID into md_type and pk_type - * - * \param oid OID to use - * \param md_alg place to store message digest algorithm - * \param pk_alg place to store public key algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_sig_alg(const mbedtls_asn1_buf *oid, - mbedtls_md_type_t *md_alg, mbedtls_pk_type_t *pk_alg); - -/** - * \brief Translate SignatureAlgorithm OID into description - * - * \param oid OID to use - * \param desc place to store string pointer - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_sig_alg_desc(const mbedtls_asn1_buf *oid, const char **desc); - -/** - * \brief Translate md_type and pk_type into SignatureAlgorithm OID - * - * \param md_alg message digest algorithm - * \param pk_alg public key algorithm - * \param oid place to store ASN.1 OID string pointer - * \param olen length of the OID - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_oid_by_sig_alg(mbedtls_pk_type_t pk_alg, mbedtls_md_type_t md_alg, - const char **oid, size_t *olen); - -/** - * \brief Translate hmac algorithm OID into md_type - * - * \param oid OID to use - * \param md_hmac place to store message hmac algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_md_hmac(const mbedtls_asn1_buf *oid, mbedtls_md_type_t *md_hmac); - -/** - * \brief Translate hash algorithm OID into md_type - * - * \param oid OID to use - * \param md_alg place to store message digest algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_md_alg(const mbedtls_asn1_buf *oid, mbedtls_md_type_t *md_alg); - -#if !defined(MBEDTLS_X509_REMOVE_INFO) -/** - * \brief Translate Extended Key Usage OID into description - * - * \param oid OID to use - * \param desc place to store string pointer - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_extended_key_usage(const mbedtls_asn1_buf *oid, const char **desc); -#endif - -/** - * \brief Translate certificate policies OID into description - * - * \param oid OID to use - * \param desc place to store string pointer - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_certificate_policies(const mbedtls_asn1_buf *oid, const char **desc); - -/** - * \brief Translate md_type into hash algorithm OID - * - * \param md_alg message digest algorithm - * \param oid place to store ASN.1 OID string pointer - * \param olen length of the OID - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_oid_by_md(mbedtls_md_type_t md_alg, const char **oid, size_t *olen); - -#if defined(MBEDTLS_CIPHER_C) -/** - * \brief Translate encryption algorithm OID into cipher_type - * - * \param oid OID to use - * \param cipher_alg place to store cipher algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_cipher_alg(const mbedtls_asn1_buf *oid, mbedtls_cipher_type_t *cipher_alg); - -#if defined(MBEDTLS_PKCS12_C) -/** - * \brief Translate PKCS#12 PBE algorithm OID into md_type and - * cipher_type - * - * \param oid OID to use - * \param md_alg place to store message digest algorithm - * \param cipher_alg place to store cipher algorithm - * - * \return 0 if successful, or MBEDTLS_ERR_OID_NOT_FOUND - */ -int mbedtls_oid_get_pkcs12_pbe_alg(const mbedtls_asn1_buf *oid, mbedtls_md_type_t *md_alg, - mbedtls_cipher_type_t *cipher_alg); -#endif /* MBEDTLS_PKCS12_C */ -#endif /* MBEDTLS_CIPHER_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* oid.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/pem.h b/android/mbedtls/arm64-v8a/include/mbedtls/pem.h deleted file mode 100644 index 3c6a28d98..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/pem.h +++ /dev/null @@ -1,160 +0,0 @@ -/** - * \file pem.h - * - * \brief Privacy Enhanced Mail (PEM) decoding - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PEM_H -#define MBEDTLS_PEM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include - -/** - * \name PEM Error codes - * These error codes are returned in case of errors reading the - * PEM data. - * \{ - */ -/** No PEM header or footer found. */ -#define MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT -0x1080 -/** PEM string is not as expected. */ -#define MBEDTLS_ERR_PEM_INVALID_DATA -0x1100 -/** Failed to allocate memory. */ -#define MBEDTLS_ERR_PEM_ALLOC_FAILED -0x1180 -/** RSA IV is not in hex-format. */ -#define MBEDTLS_ERR_PEM_INVALID_ENC_IV -0x1200 -/** Unsupported key encryption algorithm. */ -#define MBEDTLS_ERR_PEM_UNKNOWN_ENC_ALG -0x1280 -/** Private key password can't be empty. */ -#define MBEDTLS_ERR_PEM_PASSWORD_REQUIRED -0x1300 -/** Given private key password does not allow for correct decryption. */ -#define MBEDTLS_ERR_PEM_PASSWORD_MISMATCH -0x1380 -/** Unavailable feature, e.g. hashing/encryption combination. */ -#define MBEDTLS_ERR_PEM_FEATURE_UNAVAILABLE -0x1400 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_PEM_BAD_INPUT_DATA -0x1480 -/** \} name PEM Error codes */ - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(MBEDTLS_PEM_PARSE_C) -/** - * \brief PEM context structure - */ -typedef struct mbedtls_pem_context { - unsigned char *MBEDTLS_PRIVATE(buf); /*!< buffer for decoded data */ - size_t MBEDTLS_PRIVATE(buflen); /*!< length of the buffer */ - unsigned char *MBEDTLS_PRIVATE(info); /*!< buffer for extra header information */ -} -mbedtls_pem_context; - -/** - * \brief PEM context setup - * - * \param ctx context to be initialized - */ -void mbedtls_pem_init(mbedtls_pem_context *ctx); - -/** - * \brief Read a buffer for PEM information and store the resulting - * data into the specified context buffers. - * - * \param ctx context to use - * \param header header string to seek and expect - * \param footer footer string to seek and expect - * \param data source data to look in (must be nul-terminated) - * \param pwd password for decryption (can be NULL) - * \param pwdlen length of password - * \param use_len destination for total length used from data buffer. It is - * set after header is correctly read, so unless you get - * MBEDTLS_ERR_PEM_BAD_INPUT_DATA or - * MBEDTLS_ERR_PEM_NO_HEADER_FOOTER_PRESENT, use_len is - * the length to skip. - * - * \note Attempts to check password correctness by verifying if - * the decrypted text starts with an ASN.1 sequence of - * appropriate length - * - * \note \c mbedtls_pem_free must be called on PEM context before - * the PEM context can be reused in another call to - * \c mbedtls_pem_read_buffer - * - * \return 0 on success, or a specific PEM error code - */ -int mbedtls_pem_read_buffer(mbedtls_pem_context *ctx, const char *header, const char *footer, - const unsigned char *data, - const unsigned char *pwd, - size_t pwdlen, size_t *use_len); - -/** - * \brief Get the pointer to the decoded binary data in a PEM context. - * - * \param ctx PEM context to access. - * \param buflen On success, this will contain the length of the binary data. - * This must be a valid (non-null) pointer. - * - * \return A pointer to the decoded binary data. - * - * \note The returned pointer remains valid only until \p ctx is - modified or freed. - */ -static inline const unsigned char *mbedtls_pem_get_buffer(mbedtls_pem_context *ctx, size_t *buflen) -{ - *buflen = ctx->MBEDTLS_PRIVATE(buflen); - return ctx->MBEDTLS_PRIVATE(buf); -} - - -/** - * \brief PEM context memory freeing - * - * \param ctx context to be freed - */ -void mbedtls_pem_free(mbedtls_pem_context *ctx); -#endif /* MBEDTLS_PEM_PARSE_C */ - -#if defined(MBEDTLS_PEM_WRITE_C) -/** - * \brief Write a buffer of PEM information from a DER encoded - * buffer. - * - * \param header The header string to write. - * \param footer The footer string to write. - * \param der_data The DER data to encode. - * \param der_len The length of the DER data \p der_data in Bytes. - * \param buf The buffer to write to. - * \param buf_len The length of the output buffer \p buf in Bytes. - * \param olen The address at which to store the total length written - * or required (if \p buf_len is not enough). - * - * \note You may pass \c NULL for \p buf and \c 0 for \p buf_len - * to request the length of the resulting PEM buffer in - * `*olen`. - * - * \note This function may be called with overlapping \p der_data - * and \p buf buffers. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL if \p buf isn't large - * enough to hold the PEM buffer. In this case, `*olen` holds - * the required minimum size of \p buf. - * \return Another PEM or BASE64 error code on other kinds of failure. - */ -int mbedtls_pem_write_buffer(const char *header, const char *footer, - const unsigned char *der_data, size_t der_len, - unsigned char *buf, size_t buf_len, size_t *olen); -#endif /* MBEDTLS_PEM_WRITE_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* pem.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/pk.h b/android/mbedtls/arm64-v8a/include/mbedtls/pk.h deleted file mode 100644 index 2b7f34bc0..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/pk.h +++ /dev/null @@ -1,1296 +0,0 @@ -/** - * \file pk.h - * - * \brief Public Key abstraction layer - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_PK_H -#define MBEDTLS_PK_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/md.h" - -#if defined(MBEDTLS_RSA_C) -#include "mbedtls/rsa.h" -#endif - -#if defined(MBEDTLS_ECP_C) -#include "mbedtls/ecp.h" -#endif - -#if defined(MBEDTLS_ECDSA_C) -#include "mbedtls/ecdsa.h" -#endif - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) -#include "psa/crypto.h" -#endif - -/** Memory allocation failed. */ -#define MBEDTLS_ERR_PK_ALLOC_FAILED -0x3F80 -/** Type mismatch, eg attempt to encrypt with an ECDSA key */ -#define MBEDTLS_ERR_PK_TYPE_MISMATCH -0x3F00 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_PK_BAD_INPUT_DATA -0x3E80 -/** Read/write of file failed. */ -#define MBEDTLS_ERR_PK_FILE_IO_ERROR -0x3E00 -/** Unsupported key version */ -#define MBEDTLS_ERR_PK_KEY_INVALID_VERSION -0x3D80 -/** Invalid key tag or value. */ -#define MBEDTLS_ERR_PK_KEY_INVALID_FORMAT -0x3D00 -/** Key algorithm is unsupported (only RSA and EC are supported). */ -#define MBEDTLS_ERR_PK_UNKNOWN_PK_ALG -0x3C80 -/** Private key password can't be empty. */ -#define MBEDTLS_ERR_PK_PASSWORD_REQUIRED -0x3C00 -/** Given private key password does not allow for correct decryption. */ -#define MBEDTLS_ERR_PK_PASSWORD_MISMATCH -0x3B80 -/** The pubkey tag or value is invalid (only RSA and EC are supported). */ -#define MBEDTLS_ERR_PK_INVALID_PUBKEY -0x3B00 -/** The algorithm tag or value is invalid. */ -#define MBEDTLS_ERR_PK_INVALID_ALG -0x3A80 -/** Elliptic curve is unsupported (only NIST curves are supported). */ -#define MBEDTLS_ERR_PK_UNKNOWN_NAMED_CURVE -0x3A00 -/** Unavailable feature, e.g. RSA disabled for RSA key. */ -#define MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE -0x3980 -/** The buffer contains a valid signature followed by more data. */ -#define MBEDTLS_ERR_PK_SIG_LEN_MISMATCH -0x3900 -/** The output buffer is too small. */ -#define MBEDTLS_ERR_PK_BUFFER_TOO_SMALL -0x3880 - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Public key types - */ -typedef enum { - MBEDTLS_PK_NONE=0, - MBEDTLS_PK_RSA, - MBEDTLS_PK_ECKEY, - MBEDTLS_PK_ECKEY_DH, - MBEDTLS_PK_ECDSA, - MBEDTLS_PK_RSA_ALT, - MBEDTLS_PK_RSASSA_PSS, - MBEDTLS_PK_OPAQUE, -} mbedtls_pk_type_t; - -/** - * \brief Options for RSASSA-PSS signature verification. - * See \c mbedtls_rsa_rsassa_pss_verify_ext() - */ -typedef struct mbedtls_pk_rsassa_pss_options { - /** The digest to use for MGF1 in PSS. - * - * \note When #MBEDTLS_USE_PSA_CRYPTO is enabled and #MBEDTLS_RSA_C is - * disabled, this must be equal to the \c md_alg argument passed - * to mbedtls_pk_verify_ext(). In a future version of the library, - * this constraint may apply whenever #MBEDTLS_USE_PSA_CRYPTO is - * enabled regardless of the status of #MBEDTLS_RSA_C. - */ - mbedtls_md_type_t mgf1_hash_id; - - /** The expected length of the salt, in bytes. This may be - * #MBEDTLS_RSA_SALT_LEN_ANY to accept any salt length. - * - * \note When #MBEDTLS_USE_PSA_CRYPTO is enabled, only - * #MBEDTLS_RSA_SALT_LEN_ANY is valid. Any other value may be - * ignored (allowing any salt length). - */ - int expected_salt_len; - -} mbedtls_pk_rsassa_pss_options; - -/** - * \brief Maximum size of a signature made by mbedtls_pk_sign(). - */ -/* We need to set MBEDTLS_PK_SIGNATURE_MAX_SIZE to the maximum signature - * size among the supported signature types. Do it by starting at 0, - * then incrementally increasing to be large enough for each supported - * signature mechanism. - * - * The resulting value can be 0, for example if MBEDTLS_ECDH_C is enabled - * (which allows the pk module to be included) but neither MBEDTLS_ECDSA_C - * nor MBEDTLS_RSA_C nor any opaque signature mechanism (PSA or RSA_ALT). - */ -#define MBEDTLS_PK_SIGNATURE_MAX_SIZE 0 - -#if (defined(MBEDTLS_RSA_C) || defined(MBEDTLS_PK_RSA_ALT_SUPPORT)) && \ - MBEDTLS_MPI_MAX_SIZE > MBEDTLS_PK_SIGNATURE_MAX_SIZE -/* For RSA, the signature can be as large as the bignum module allows. - * For RSA_ALT, the signature size is not necessarily tied to what the - * bignum module can do, but in the absence of any specific setting, - * we use that (rsa_alt_sign_wrap in library/pk_wrap.h will check). */ -#undef MBEDTLS_PK_SIGNATURE_MAX_SIZE -#define MBEDTLS_PK_SIGNATURE_MAX_SIZE MBEDTLS_MPI_MAX_SIZE -#endif - -#if defined(MBEDTLS_ECDSA_C) && \ - MBEDTLS_ECDSA_MAX_LEN > MBEDTLS_PK_SIGNATURE_MAX_SIZE -/* For ECDSA, the ecdsa module exports a constant for the maximum - * signature size. */ -#undef MBEDTLS_PK_SIGNATURE_MAX_SIZE -#define MBEDTLS_PK_SIGNATURE_MAX_SIZE MBEDTLS_ECDSA_MAX_LEN -#endif - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -#if PSA_SIGNATURE_MAX_SIZE > MBEDTLS_PK_SIGNATURE_MAX_SIZE -/* PSA_SIGNATURE_MAX_SIZE is the maximum size of a signature made - * through the PSA API in the PSA representation. */ -#undef MBEDTLS_PK_SIGNATURE_MAX_SIZE -#define MBEDTLS_PK_SIGNATURE_MAX_SIZE PSA_SIGNATURE_MAX_SIZE -#endif - -#if PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE + 11 > MBEDTLS_PK_SIGNATURE_MAX_SIZE -/* The Mbed TLS representation is different for ECDSA signatures: - * PSA uses the raw concatenation of r and s, - * whereas Mbed TLS uses the ASN.1 representation (SEQUENCE of two INTEGERs). - * Add the overhead of ASN.1: up to (1+2) + 2 * (1+2+1) for the - * types, lengths (represented by up to 2 bytes), and potential leading - * zeros of the INTEGERs and the SEQUENCE. */ -#undef MBEDTLS_PK_SIGNATURE_MAX_SIZE -#define MBEDTLS_PK_SIGNATURE_MAX_SIZE (PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE + 11) -#endif -#endif /* defined(MBEDTLS_USE_PSA_CRYPTO) */ - -/* Internal helper to define which fields in the pk_context structure below - * should be used for EC keys: legacy ecp_keypair or the raw (PSA friendly) - * format. It should be noted that this only affects how data is stored, not - * which functions are used for various operations. The overall picture looks - * like this: - * - if USE_PSA is not defined and ECP_C is defined then use ecp_keypair data - * structure and legacy functions - * - if USE_PSA is defined and - * - if ECP_C then use ecp_keypair structure, convert data to a PSA friendly - * format and use PSA functions - * - if !ECP_C then use new raw data and PSA functions directly. - * - * The main reason for the "intermediate" (USE_PSA + ECP_C) above is that as long - * as ECP_C is defined mbedtls_pk_ec() gives the user a read/write access to the - * ecp_keypair structure inside the pk_context so they can modify it using - * ECP functions which are not under PK module's control. - */ -#if defined(MBEDTLS_USE_PSA_CRYPTO) && defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) && \ - !defined(MBEDTLS_ECP_C) -#define MBEDTLS_PK_USE_PSA_EC_DATA -#endif - -/** - * \brief Types for interfacing with the debug module - */ -typedef enum { - MBEDTLS_PK_DEBUG_NONE = 0, - MBEDTLS_PK_DEBUG_MPI, - MBEDTLS_PK_DEBUG_ECP, - MBEDTLS_PK_DEBUG_PSA_EC, -} mbedtls_pk_debug_type; - -/** - * \brief Item to send to the debug module - */ -typedef struct mbedtls_pk_debug_item { - mbedtls_pk_debug_type MBEDTLS_PRIVATE(type); - const char *MBEDTLS_PRIVATE(name); - void *MBEDTLS_PRIVATE(value); -} mbedtls_pk_debug_item; - -/** Maximum number of item send for debugging, plus 1 */ -#define MBEDTLS_PK_DEBUG_MAX_ITEMS 3 - -/** - * \brief Public key information and operations - * - * \note The library does not support custom pk info structures, - * only built-in structures returned by - * mbedtls_cipher_info_from_type(). - */ -typedef struct mbedtls_pk_info_t mbedtls_pk_info_t; - -#define MBEDTLS_PK_MAX_EC_PUBKEY_RAW_LEN \ - PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) -/** - * \brief Public key container - */ -typedef struct mbedtls_pk_context { - const mbedtls_pk_info_t *MBEDTLS_PRIVATE(pk_info); /**< Public key information */ - void *MBEDTLS_PRIVATE(pk_ctx); /**< Underlying public key context */ - /* The following field is used to store the ID of a private key in the - * following cases: - * - opaque key when MBEDTLS_USE_PSA_CRYPTO is defined - * - normal key when MBEDTLS_PK_USE_PSA_EC_DATA is defined. In this case: - * - the pk_ctx above is not not used to store the private key anymore. - * Actually that field not populated at all in this case because also - * the public key will be stored in raw format as explained below - * - this ID is used for all private key operations (ex: sign, check - * key pair, key write, etc) using PSA functions - * - * Note: this private key storing solution only affects EC keys, not the - * other ones. The latters still use the pk_ctx to store their own - * context. */ -#if defined(MBEDTLS_USE_PSA_CRYPTO) - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(priv_id); /**< Key ID for opaque keys */ -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - /* The following fields are meant for storing the public key in raw format - * which is handy for: - * - easily importing it into the PSA context - * - reducing the ECP module dependencies in the PK one. - * - * When MBEDTLS_PK_USE_PSA_EC_DATA is enabled: - * - the pk_ctx above is not used anymore for storing the public key - * inside the ecp_keypair structure - * - the following fields are used for all public key operations: signature - * verify, key pair check and key write. - * - For a key pair, priv_id contains the private key. For a public key, - * priv_id is null. - * Of course, when MBEDTLS_PK_USE_PSA_EC_DATA is not enabled, the legacy - * ecp_keypair structure is used for storing the public key and performing - * all the operations. - * - * Note: This new public key storing solution only works for EC keys, not - * other ones. The latters still use pk_ctx to store their own - * context. - */ -#if defined(MBEDTLS_PK_USE_PSA_EC_DATA) - uint8_t MBEDTLS_PRIVATE(pub_raw)[MBEDTLS_PK_MAX_EC_PUBKEY_RAW_LEN]; /**< Raw public key */ - size_t MBEDTLS_PRIVATE(pub_raw_len); /**< Valid bytes in "pub_raw" */ - psa_ecc_family_t MBEDTLS_PRIVATE(ec_family); /**< EC family of pk */ - size_t MBEDTLS_PRIVATE(ec_bits); /**< Curve's bits of pk */ -#endif /* MBEDTLS_PK_USE_PSA_EC_DATA */ -} mbedtls_pk_context; - -#if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Context for resuming operations - */ -typedef struct { - const mbedtls_pk_info_t *MBEDTLS_PRIVATE(pk_info); /**< Public key information */ - void *MBEDTLS_PRIVATE(rs_ctx); /**< Underlying restart context */ -} mbedtls_pk_restart_ctx; -#else /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ -/* Now we can declare functions that take a pointer to that */ -typedef void mbedtls_pk_restart_ctx; -#endif /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ - -#if defined(MBEDTLS_PK_RSA_ALT_SUPPORT) -/** - * \brief Types for RSA-alt abstraction - */ -typedef int (*mbedtls_pk_rsa_alt_decrypt_func)(void *ctx, size_t *olen, - const unsigned char *input, unsigned char *output, - size_t output_max_len); -typedef int (*mbedtls_pk_rsa_alt_sign_func)(void *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_md_type_t md_alg, unsigned int hashlen, - const unsigned char *hash, unsigned char *sig); -typedef size_t (*mbedtls_pk_rsa_alt_key_len_func)(void *ctx); -#endif /* MBEDTLS_PK_RSA_ALT_SUPPORT */ - -/** - * \brief Return information associated with the given PK type - * - * \param pk_type PK type to search for. - * - * \return The PK info associated with the type or NULL if not found. - */ -const mbedtls_pk_info_t *mbedtls_pk_info_from_type(mbedtls_pk_type_t pk_type); - -/** - * \brief Initialize a #mbedtls_pk_context (as NONE). - * - * \param ctx The context to initialize. - * This must not be \c NULL. - */ -void mbedtls_pk_init(mbedtls_pk_context *ctx); - -/** - * \brief Free the components of a #mbedtls_pk_context. - * - * \param ctx The context to clear. It must have been initialized. - * If this is \c NULL, this function does nothing. - * - * \note For contexts that have been set up with - * mbedtls_pk_setup_opaque(), this does not free the underlying - * PSA key and you still need to call psa_destroy_key() - * independently if you want to destroy that key. - */ -void mbedtls_pk_free(mbedtls_pk_context *ctx); - -#if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Initialize a restart context - * - * \param ctx The context to initialize. - * This must not be \c NULL. - */ -void mbedtls_pk_restart_init(mbedtls_pk_restart_ctx *ctx); - -/** - * \brief Free the components of a restart context - * - * \param ctx The context to clear. It must have been initialized. - * If this is \c NULL, this function does nothing. - */ -void mbedtls_pk_restart_free(mbedtls_pk_restart_ctx *ctx); -#endif /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ - -/** - * \brief Initialize a PK context with the information given - * and allocates the type-specific PK subcontext. - * - * \param ctx Context to initialize. It must not have been set - * up yet (type #MBEDTLS_PK_NONE). - * \param info Information to use - * - * \return 0 on success, - * MBEDTLS_ERR_PK_BAD_INPUT_DATA on invalid input, - * MBEDTLS_ERR_PK_ALLOC_FAILED on allocation failure. - * - * \note For contexts holding an RSA-alt key, use - * \c mbedtls_pk_setup_rsa_alt() instead. - */ -int mbedtls_pk_setup(mbedtls_pk_context *ctx, const mbedtls_pk_info_t *info); - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -/** - * \brief Initialize a PK context to wrap a PSA key. - * - * This function creates a PK context which wraps a PSA key. The PSA wrapped - * key must be an EC or RSA key pair (DH is not suported in the PK module). - * - * Under the hood PSA functions will be used to perform the required - * operations and, based on the key type, used algorithms will be: - * * EC: - * * verify, verify_ext, sign, sign_ext: ECDSA. - * * RSA: - * * sign, decrypt: use the primary algorithm in the wrapped PSA key; - * * sign_ext: RSA PSS if the pk_type is #MBEDTLS_PK_RSASSA_PSS, otherwise - * it falls back to the sign() case; - * * verify, verify_ext, encrypt: not supported. - * - * In order for the above operations to succeed, the policy of the wrapped PSA - * key must allow the specified algorithm. - * - * Opaque PK contexts wrapping an EC keys also support \c mbedtls_pk_check_pair(), - * whereas RSA ones do not. - * - * \warning The PSA wrapped key must remain valid as long as the wrapping PK - * context is in use, that is at least between the point this function - * is called and the point mbedtls_pk_free() is called on this context. - * - * \param ctx The context to initialize. It must be empty (type NONE). - * \param key The PSA key to wrap, which must hold an ECC or RSA key pair. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_PK_BAD_INPUT_DATA on invalid input (context already - * used, invalid key identifier). - * \return #MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE if the key is not an ECC or - * RSA key pair. - * \return #MBEDTLS_ERR_PK_ALLOC_FAILED on allocation failure. - */ -int mbedtls_pk_setup_opaque(mbedtls_pk_context *ctx, - const mbedtls_svc_key_id_t key); -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -#if defined(MBEDTLS_PK_RSA_ALT_SUPPORT) -/** - * \brief Initialize an RSA-alt context - * - * \param ctx Context to initialize. It must not have been set - * up yet (type #MBEDTLS_PK_NONE). - * \param key RSA key pointer - * \param decrypt_func Decryption function - * \param sign_func Signing function - * \param key_len_func Function returning key length in bytes - * - * \return 0 on success, or MBEDTLS_ERR_PK_BAD_INPUT_DATA if the - * context wasn't already initialized as RSA_ALT. - * - * \note This function replaces \c mbedtls_pk_setup() for RSA-alt. - */ -int mbedtls_pk_setup_rsa_alt(mbedtls_pk_context *ctx, void *key, - mbedtls_pk_rsa_alt_decrypt_func decrypt_func, - mbedtls_pk_rsa_alt_sign_func sign_func, - mbedtls_pk_rsa_alt_key_len_func key_len_func); -#endif /* MBEDTLS_PK_RSA_ALT_SUPPORT */ - -/** - * \brief Get the size in bits of the underlying key - * - * \param ctx The context to query. It must have been initialized. - * - * \return Key size in bits, or 0 on error - */ -size_t mbedtls_pk_get_bitlen(const mbedtls_pk_context *ctx); - -/** - * \brief Get the length in bytes of the underlying key - * - * \param ctx The context to query. It must have been initialized. - * - * \return Key length in bytes, or 0 on error - */ -static inline size_t mbedtls_pk_get_len(const mbedtls_pk_context *ctx) -{ - return (mbedtls_pk_get_bitlen(ctx) + 7) / 8; -} - -/** - * \brief Tell if a context can do the operation given by type - * - * \param ctx The context to query. It must have been initialized. - * \param type The desired type. - * - * \return 1 if the context can do operations on the given type. - * \return 0 if the context cannot do the operations on the given - * type. This is always the case for a context that has - * been initialized but not set up, or that has been - * cleared with mbedtls_pk_free(). - */ -int mbedtls_pk_can_do(const mbedtls_pk_context *ctx, mbedtls_pk_type_t type); - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -/** - * \brief Tell if context can do the operation given by PSA algorithm - * - * \param ctx The context to query. It must have been initialized. - * \param alg PSA algorithm to check against, the following are allowed: - * PSA_ALG_RSA_PKCS1V15_SIGN(hash), - * PSA_ALG_RSA_PSS(hash), - * PSA_ALG_RSA_PKCS1V15_CRYPT, - * PSA_ALG_ECDSA(hash), - * PSA_ALG_ECDH, where hash is a specific hash. - * \param usage PSA usage flag to check against, must be composed of: - * PSA_KEY_USAGE_SIGN_HASH - * PSA_KEY_USAGE_DECRYPT - * PSA_KEY_USAGE_DERIVE. - * Context key must match all passed usage flags. - * - * \warning Since the set of allowed algorithms and usage flags may be - * expanded in the future, the return value \c 0 should not - * be taken in account for non-allowed algorithms and usage - * flags. - * - * \return 1 if the context can do operations on the given type. - * \return 0 if the context cannot do the operations on the given - * type, for non-allowed algorithms and usage flags, or - * for a context that has been initialized but not set up - * or that has been cleared with mbedtls_pk_free(). - */ -int mbedtls_pk_can_do_ext(const mbedtls_pk_context *ctx, psa_algorithm_t alg, - psa_key_usage_t usage); -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) -/** - * \brief Determine valid PSA attributes that can be used to - * import a key into PSA. - * - * The attributes determined by this function are suitable - * for calling mbedtls_pk_import_into_psa() to create - * a PSA key with the same key material. - * - * The typical flow of operations involving this function is - * ``` - * psa_key_attributes_t attributes = PSA_KEY_ATTRIBUTES_INIT; - * int ret = mbedtls_pk_get_psa_attributes(pk, &attributes); - * if (ret != 0) ...; // error handling omitted - * // Tweak attributes if desired - * psa_key_id_t key_id = 0; - * ret = mbedtls_pk_import_into_psa(pk, &attributes, &key_id); - * if (ret != 0) ...; // error handling omitted - * ``` - * - * \note This function does not support RSA-alt contexts - * (set up with mbedtls_pk_setup_rsa_alt()). - * - * \param[in] pk The PK context to use. It must have been set up. - * It can either contain a key pair or just a public key. - * \param usage A single `PSA_KEY_USAGE_xxx` flag among the following: - * - #PSA_KEY_USAGE_DECRYPT: \p pk must contain a - * key pair. The output \p attributes will contain a - * key pair type, and the usage policy will allow - * #PSA_KEY_USAGE_ENCRYPT as well as - * #PSA_KEY_USAGE_DECRYPT. - * - #PSA_KEY_USAGE_DERIVE: \p pk must contain a - * key pair. The output \p attributes will contain a - * key pair type. - * - #PSA_KEY_USAGE_ENCRYPT: The output - * \p attributes will contain a public key type. - * - #PSA_KEY_USAGE_SIGN_HASH: \p pk must contain a - * key pair. The output \p attributes will contain a - * key pair type, and the usage policy will allow - * #PSA_KEY_USAGE_VERIFY_HASH as well as - * #PSA_KEY_USAGE_SIGN_HASH. - * - #PSA_KEY_USAGE_SIGN_MESSAGE: \p pk must contain a - * key pair. The output \p attributes will contain a - * key pair type, and the usage policy will allow - * #PSA_KEY_USAGE_VERIFY_MESSAGE as well as - * #PSA_KEY_USAGE_SIGN_MESSAGE. - * - #PSA_KEY_USAGE_VERIFY_HASH: The output - * \p attributes will contain a public key type. - * - #PSA_KEY_USAGE_VERIFY_MESSAGE: The output - * \p attributes will contain a public key type. - * \param[out] attributes - * On success, valid attributes to import the key into PSA. - * - The lifetime and key identifier are unchanged. If the - * attribute structure was initialized or reset before - * calling this function, this will result in a volatile - * key. Call psa_set_key_identifier() before or after this - * function if you wish to create a persistent key. Call - * psa_set_key_lifetime() before or after this function if - * you wish to import the key in a secure element. - * - The key type and bit-size are determined by the contents - * of the PK context. If the PK context contains a key - * pair, the key type can be either a key pair type or - * the corresponding public key type, depending on - * \p usage. If the PK context contains a public key, - * the key type is a public key type. - * - The key's policy is determined by the key type and - * the \p usage parameter. The usage always allows - * \p usage, exporting and copying the key, and - * possibly other permissions as documented for the - * \p usage parameter. - * The permitted algorithm policy is determined as follows - * based on the #mbedtls_pk_type_t type of \p pk, - * the chosen \p usage and other factors: - * - #MBEDTLS_PK_RSA whose underlying - * #mbedtls_rsa_context has the padding mode - * #MBEDTLS_RSA_PKCS_V15: - * #PSA_ALG_RSA_PKCS1V15_SIGN(#PSA_ALG_ANY_HASH) - * if \p usage is SIGN/VERIFY, and - * #PSA_ALG_RSA_PKCS1V15_CRYPT - * if \p usage is ENCRYPT/DECRYPT. - * - #MBEDTLS_PK_RSA whose underlying - * #mbedtls_rsa_context has the padding mode - * #MBEDTLS_RSA_PKCS_V21 and the digest type - * corresponding to the PSA algorithm \c hash: - * #PSA_ALG_RSA_PSS_ANY_SALT(#PSA_ALG_ANY_HASH) - * if \p usage is SIGN/VERIFY, and - * #PSA_ALG_RSA_OAEP(\c hash) - * if \p usage is ENCRYPT/DECRYPT. - * - #MBEDTLS_PK_RSA_ALT: not supported. - * - #MBEDTLS_PK_ECDSA or #MBEDTLS_PK_ECKEY - * if \p usage is SIGN/VERIFY: - * #PSA_ALG_DETERMINISTIC_ECDSA(#PSA_ALG_ANY_HASH) - * if #MBEDTLS_ECDSA_DETERMINISTIC is enabled, - * otherwise #PSA_ALG_ECDSA(#PSA_ALG_ANY_HASH). - * - #MBEDTLS_PK_ECKEY_DH or #MBEDTLS_PK_ECKEY - * if \p usage is DERIVE: - * #PSA_ALG_ECDH. - * - #MBEDTLS_PK_OPAQUE: same as the primary algorithm - * set for the underlying PSA key, except that - * sign/decrypt flags are removed if the type is - * set to a public key type. - * The underlying key must allow \p usage. - * Note that the enrollment algorithm set with - * psa_set_key_enrollment_algorithm() is not copied. - * - * \return 0 on success. - * #MBEDTLS_ERR_PK_TYPE_MISMATCH if \p pk does not contain - * a key of the type identified in \p attributes. - * Another error code on other failures. - */ -int mbedtls_pk_get_psa_attributes(const mbedtls_pk_context *pk, - psa_key_usage_t usage, - psa_key_attributes_t *attributes); - -/** - * \brief Import a key into the PSA key store. - * - * This function is equivalent to calling psa_import_key() - * with the key material from \p pk. - * - * The typical way to use this function is: - * -# Call mbedtls_pk_get_psa_attributes() to obtain - * attributes for the given key. - * -# If desired, modify the attributes, for example: - * - To create a persistent key, call - * psa_set_key_identifier() and optionally - * psa_set_key_lifetime(). - * - To import only the public part of a key pair: - * - * psa_set_key_type(&attributes, - * PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR( - * psa_get_key_type(&attributes))); - * - Restrict the key usage if desired. - * -# Call mbedtls_pk_import_into_psa(). - * - * \note This function does not support RSA-alt contexts - * (set up with mbedtls_pk_setup_rsa_alt()). - * - * \param[in] pk The PK context to use. It must have been set up. - * It can either contain a key pair or just a public key. - * \param[in] attributes - * The attributes to use for the new key. They must be - * compatible with \p pk. In particular, the key type - * must match the content of \p pk. - * If \p pk contains a key pair, the key type in - * attributes can be either the key pair type or the - * corresponding public key type (to import only the - * public part). - * \param[out] key_id - * On success, the identifier of the newly created key. - * On error, this is #MBEDTLS_SVC_KEY_ID_INIT. - * - * \return 0 on success. - * #MBEDTLS_ERR_PK_TYPE_MISMATCH if \p pk does not contain - * a key of the type identified in \p attributes. - * Another error code on other failures. - */ -int mbedtls_pk_import_into_psa(const mbedtls_pk_context *pk, - const psa_key_attributes_t *attributes, - mbedtls_svc_key_id_t *key_id); - -/** - * \brief Create a PK context starting from a key stored in PSA. - * This key: - * - must be exportable and - * - must be an RSA or EC key pair or public key (FFDH is not supported in PK). - * - * The resulting PK object will be a transparent type: - * - #MBEDTLS_PK_RSA for RSA keys or - * - #MBEDTLS_PK_ECKEY for EC keys. - * - * Once this functions returns the PK object will be completely - * independent from the original PSA key that it was generated - * from. - * Calling mbedtls_pk_sign(), mbedtls_pk_verify(), - * mbedtls_pk_encrypt(), mbedtls_pk_decrypt() on the resulting - * PK context will perform the corresponding algorithm for that - * PK context type. - * * For ECDSA, the choice of deterministic vs randomized will - * be based on the compile-time setting #MBEDTLS_ECDSA_DETERMINISTIC. - * * For an RSA key, the output PK context will allow both - * encrypt/decrypt and sign/verify regardless of the original - * key's policy. - * The original key's policy determines the output key's padding - * mode: PCKS1 v2.1 is set if the PSA key policy is OAEP or PSS, - * otherwise PKCS1 v1.5 is set. - * - * \param key_id The key identifier of the key stored in PSA. - * \param pk The PK context that will be filled. It must be initialized, - * but not set up. - * - * \return 0 on success. - * \return #MBEDTLS_ERR_PK_BAD_INPUT_DATA in case the provided input - * parameters are not correct. - */ -int mbedtls_pk_copy_from_psa(mbedtls_svc_key_id_t key_id, mbedtls_pk_context *pk); - -/** - * \brief Create a PK context for the public key of a PSA key. - * - * The key must be an RSA or ECC key. It can be either a - * public key or a key pair, and only the public key is copied. - * The resulting PK object will be a transparent type: - * - #MBEDTLS_PK_RSA for RSA keys or - * - #MBEDTLS_PK_ECKEY for EC keys. - * - * Once this functions returns the PK object will be completely - * independent from the original PSA key that it was generated - * from. - * Calling mbedtls_pk_verify() or - * mbedtls_pk_encrypt() on the resulting - * PK context will perform the corresponding algorithm for that - * PK context type. - * - * For an RSA key, the output PK context will allow both - * encrypt and verify regardless of the original key's policy. - * The original key's policy determines the output key's padding - * mode: PCKS1 v2.1 is set if the PSA key policy is OAEP or PSS, - * otherwise PKCS1 v1.5 is set. - * - * \param key_id The key identifier of the key stored in PSA. - * \param pk The PK context that will be filled. It must be initialized, - * but not set up. - * - * \return 0 on success. - * \return MBEDTLS_ERR_PK_BAD_INPUT_DATA in case the provided input - * parameters are not correct. - */ -int mbedtls_pk_copy_public_from_psa(mbedtls_svc_key_id_t key_id, mbedtls_pk_context *pk); -#endif /* MBEDTLS_PSA_CRYPTO_CLIENT */ - -/** - * \brief Verify signature (including padding if relevant). - * - * \param ctx The PK context to use. It must have been set up. - * \param md_alg Hash algorithm used. - * This can be #MBEDTLS_MD_NONE if the signature algorithm - * does not rely on a hash algorithm (non-deterministic - * ECDSA, RSA PKCS#1 v1.5). - * For PKCS#1 v1.5, if \p md_alg is #MBEDTLS_MD_NONE, then - * \p hash is the DigestInfo structure used by RFC 8017 - * §9.2 steps 3–6. If \p md_alg is a valid hash - * algorithm then \p hash is the digest itself, and this - * function calculates the DigestInfo encoding internally. - * \param hash Hash of the message to sign - * \param hash_len Hash length - * \param sig Signature to verify - * \param sig_len Signature length - * - * \note For keys of type #MBEDTLS_PK_RSA, the signature algorithm is - * either PKCS#1 v1.5 or PSS (accepting any salt length), - * depending on the padding mode in the underlying RSA context. - * For a pk object constructed by parsing, this is PKCS#1 v1.5 - * by default. Use mbedtls_pk_verify_ext() to explicitly select - * a different algorithm. - * - * \return 0 on success (signature is valid), - * #MBEDTLS_ERR_PK_SIG_LEN_MISMATCH if there is a valid - * signature in \p sig but its length is less than \p sig_len, - * or a specific error code. - */ -int mbedtls_pk_verify(mbedtls_pk_context *ctx, mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - const unsigned char *sig, size_t sig_len); - -/** - * \brief Restartable version of \c mbedtls_pk_verify() - * - * \note Performs the same job as \c mbedtls_pk_verify(), but can - * return early and restart according to the limit set with - * \c mbedtls_ecp_set_max_ops() to reduce blocking for ECC - * operations. For RSA, same as \c mbedtls_pk_verify(). - * - * \param ctx The PK context to use. It must have been set up. - * \param md_alg Hash algorithm used (see notes) - * \param hash Hash of the message to sign - * \param hash_len Hash length or 0 (see notes) - * \param sig Signature to verify - * \param sig_len Signature length - * \param rs_ctx Restart context (NULL to disable restart) - * - * \return See \c mbedtls_pk_verify(), or - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - */ -int mbedtls_pk_verify_restartable(mbedtls_pk_context *ctx, - mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - const unsigned char *sig, size_t sig_len, - mbedtls_pk_restart_ctx *rs_ctx); - -/** - * \brief Verify signature, with options. - * (Includes verification of the padding depending on type.) - * - * \param type Signature type (inc. possible padding type) to verify - * \param options Pointer to type-specific options, or NULL - * \param ctx The PK context to use. It must have been set up. - * \param md_alg Hash algorithm used (see notes) - * \param hash Hash of the message to sign - * \param hash_len Hash length or 0 (see notes) - * \param sig Signature to verify - * \param sig_len Signature length - * - * \return 0 on success (signature is valid), - * #MBEDTLS_ERR_PK_TYPE_MISMATCH if the PK context can't be - * used for this type of signatures, - * #MBEDTLS_ERR_PK_SIG_LEN_MISMATCH if there is a valid - * signature in \p sig but its length is less than \p sig_len, - * or a specific error code. - * - * \note If hash_len is 0, then the length associated with md_alg - * is used instead, or an error returned if it is invalid. - * - * \note md_alg may be MBEDTLS_MD_NONE, only if hash_len != 0 - * - * \note If type is MBEDTLS_PK_RSASSA_PSS, then options must point - * to a mbedtls_pk_rsassa_pss_options structure, - * otherwise it must be NULL. Note that if - * #MBEDTLS_USE_PSA_CRYPTO is defined, the salt length is not - * verified as PSA_ALG_RSA_PSS_ANY_SALT is used. - */ -int mbedtls_pk_verify_ext(mbedtls_pk_type_t type, const void *options, - mbedtls_pk_context *ctx, mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - const unsigned char *sig, size_t sig_len); - -/** - * \brief Make signature, including padding if relevant. - * - * \param ctx The PK context to use. It must have been set up - * with a private key. - * \param md_alg Hash algorithm used (see notes) - * \param hash Hash of the message to sign - * \param hash_len Hash length - * \param sig Place to write the signature. - * It must have enough room for the signature. - * #MBEDTLS_PK_SIGNATURE_MAX_SIZE is always enough. - * You may use a smaller buffer if it is large enough - * given the key type. - * \param sig_size The size of the \p sig buffer in bytes. - * \param sig_len On successful return, - * the number of bytes written to \p sig. - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * - * \note For keys of type #MBEDTLS_PK_RSA, the signature algorithm is - * either PKCS#1 v1.5 or PSS (using the largest possible salt - * length up to the hash length), depending on the padding mode - * in the underlying RSA context. For a pk object constructed - * by parsing, this is PKCS#1 v1.5 by default. Use - * mbedtls_pk_verify_ext() to explicitly select a different - * algorithm. - * - * \return 0 on success, or a specific error code. - * - * \note For RSA, md_alg may be MBEDTLS_MD_NONE if hash_len != 0. - * For ECDSA, md_alg may never be MBEDTLS_MD_NONE. - */ -int mbedtls_pk_sign(mbedtls_pk_context *ctx, mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - unsigned char *sig, size_t sig_size, size_t *sig_len, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** - * \brief Make signature given a signature type. - * - * \param pk_type Signature type. - * \param ctx The PK context to use. It must have been set up - * with a private key. - * \param md_alg Hash algorithm used (see notes) - * \param hash Hash of the message to sign - * \param hash_len Hash length - * \param sig Place to write the signature. - * It must have enough room for the signature. - * #MBEDTLS_PK_SIGNATURE_MAX_SIZE is always enough. - * You may use a smaller buffer if it is large enough - * given the key type. - * \param sig_size The size of the \p sig buffer in bytes. - * \param sig_len On successful return, - * the number of bytes written to \p sig. - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * - * \return 0 on success, or a specific error code. - * - * \note When \p pk_type is #MBEDTLS_PK_RSASSA_PSS, - * see #PSA_ALG_RSA_PSS for a description of PSS options used. - * - * \note For RSA, md_alg may be MBEDTLS_MD_NONE if hash_len != 0. - * For ECDSA, md_alg may never be MBEDTLS_MD_NONE. - * - */ -int mbedtls_pk_sign_ext(mbedtls_pk_type_t pk_type, - mbedtls_pk_context *ctx, - mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - unsigned char *sig, size_t sig_size, size_t *sig_len, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Restartable version of \c mbedtls_pk_sign() - * - * \note Performs the same job as \c mbedtls_pk_sign(), but can - * return early and restart according to the limit set with - * \c mbedtls_ecp_set_max_ops() to reduce blocking for ECC - * operations. For RSA, same as \c mbedtls_pk_sign(). - * - * \param ctx The PK context to use. It must have been set up - * with a private key. - * \param md_alg Hash algorithm used (see notes for mbedtls_pk_sign()) - * \param hash Hash of the message to sign - * \param hash_len Hash length - * \param sig Place to write the signature. - * It must have enough room for the signature. - * #MBEDTLS_PK_SIGNATURE_MAX_SIZE is always enough. - * You may use a smaller buffer if it is large enough - * given the key type. - * \param sig_size The size of the \p sig buffer in bytes. - * \param sig_len On successful return, - * the number of bytes written to \p sig. - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * \param rs_ctx Restart context (NULL to disable restart) - * - * \return See \c mbedtls_pk_sign(). - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - */ -int mbedtls_pk_sign_restartable(mbedtls_pk_context *ctx, - mbedtls_md_type_t md_alg, - const unsigned char *hash, size_t hash_len, - unsigned char *sig, size_t sig_size, size_t *sig_len, - mbedtls_f_rng_t *f_rng, void *p_rng, - mbedtls_pk_restart_ctx *rs_ctx); - -/** - * \brief Decrypt message (including padding if relevant). - * - * \param ctx The PK context to use. It must have been set up - * with a private key. - * \param input Input to decrypt - * \param ilen Input size - * \param output Decrypted output - * \param olen Decrypted message length - * \param osize Size of the output buffer - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * - * \note For keys of type #MBEDTLS_PK_RSA, the signature algorithm is - * either PKCS#1 v1.5 or OAEP, depending on the padding mode in - * the underlying RSA context. For a pk object constructed by - * parsing, this is PKCS#1 v1.5 by default. - * - * \return 0 on success, or a specific error code. - */ -int mbedtls_pk_decrypt(mbedtls_pk_context *ctx, - const unsigned char *input, size_t ilen, - unsigned char *output, size_t *olen, size_t osize, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** - * \brief Encrypt message (including padding if relevant). - * - * \param ctx The PK context to use. It must have been set up. - * \param input Message to encrypt - * \param ilen Message size - * \param output Encrypted output - * \param olen Encrypted output length - * \param osize Size of the output buffer - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * - * \note For keys of type #MBEDTLS_PK_RSA, the signature algorithm is - * either PKCS#1 v1.5 or OAEP, depending on the padding mode in - * the underlying RSA context. For a pk object constructed by - * parsing, this is PKCS#1 v1.5 by default. - * - * \note \p f_rng is used for padding generation. - * - * \return 0 on success, or a specific error code. - */ -int mbedtls_pk_encrypt(mbedtls_pk_context *ctx, - const unsigned char *input, size_t ilen, - unsigned char *output, size_t *olen, size_t osize, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** - * \brief Check if a public-private pair of keys matches. - * - * \param pub Context holding a public key. - * \param prv Context holding a private (and public) key. - * \param f_rng RNG function, must not be \c NULL. - * \param p_rng RNG parameter - * - * \return \c 0 on success (keys were checked and match each other). - * \return #MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE if the keys could not - * be checked - in that case they may or may not match. - * \return #MBEDTLS_ERR_PK_BAD_INPUT_DATA if a context is invalid. - * \return Another non-zero value if the keys do not match. - */ -int mbedtls_pk_check_pair(const mbedtls_pk_context *pub, - const mbedtls_pk_context *prv, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Export debug information - * - * \param ctx The PK context to use. It must have been initialized. - * \param items Place to write debug items - * - * \return 0 on success or MBEDTLS_ERR_PK_BAD_INPUT_DATA - */ -int mbedtls_pk_debug(const mbedtls_pk_context *ctx, mbedtls_pk_debug_item *items); - -/** - * \brief Access the type name - * - * \param ctx The PK context to use. It must have been initialized. - * - * \return Type name on success, or "invalid PK" - */ -const char *mbedtls_pk_get_name(const mbedtls_pk_context *ctx); - -/** - * \brief Get the key type - * - * \param ctx The PK context to use. It must have been initialized. - * - * \return Type on success. - * \return #MBEDTLS_PK_NONE for a context that has not been set up. - */ -mbedtls_pk_type_t mbedtls_pk_get_type(const mbedtls_pk_context *ctx); - -#if defined(MBEDTLS_RSA_C) -/** - * Quick access to an RSA context inside a PK context. - * - * \warning This function can only be used when the type of the context, as - * returned by mbedtls_pk_get_type(), is #MBEDTLS_PK_RSA. - * Ensuring that is the caller's responsibility. - * Alternatively, you can check whether this function returns NULL. - * - * \return The internal RSA context held by the PK context, or NULL. - */ -static inline mbedtls_rsa_context *mbedtls_pk_rsa(const mbedtls_pk_context pk) -{ - switch (mbedtls_pk_get_type(&pk)) { - case MBEDTLS_PK_RSA: - return (mbedtls_rsa_context *) (pk).MBEDTLS_PRIVATE(pk_ctx); - default: - return NULL; - } -} -#endif /* MBEDTLS_RSA_C */ - -#if defined(MBEDTLS_ECP_C) -/** - * Quick access to an EC context inside a PK context. - * - * \warning This function can only be used when the type of the context, as - * returned by mbedtls_pk_get_type(), is #MBEDTLS_PK_ECKEY, - * #MBEDTLS_PK_ECKEY_DH, or #MBEDTLS_PK_ECDSA. - * Ensuring that is the caller's responsibility. - * Alternatively, you can check whether this function returns NULL. - * - * \return The internal EC context held by the PK context, or NULL. - */ -static inline mbedtls_ecp_keypair *mbedtls_pk_ec(const mbedtls_pk_context pk) -{ - switch (mbedtls_pk_get_type(&pk)) { - case MBEDTLS_PK_ECKEY: - case MBEDTLS_PK_ECKEY_DH: - case MBEDTLS_PK_ECDSA: - return (mbedtls_ecp_keypair *) (pk).MBEDTLS_PRIVATE(pk_ctx); - default: - return NULL; - } -} -#endif /* MBEDTLS_ECP_C */ - -#if defined(MBEDTLS_PK_PARSE_C) -/** \ingroup pk_module */ -/** - * \brief Parse a private key in PEM or DER format - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param ctx The PK context to fill. It must have been initialized - * but not set up. - * \param key Input buffer to parse. - * The buffer must contain the input exactly, with no - * extra trailing material. For PEM, the buffer must - * contain a null-terminated string. - * \param keylen Size of \b key in bytes. - * For PEM data, this includes the terminating null byte, - * so \p keylen must be equal to `strlen(key) + 1`. - * \param pwd Optional password for decryption. - * Pass \c NULL if expecting a non-encrypted key. - * Pass a string of \p pwdlen bytes if expecting an encrypted - * key; a non-encrypted key will also be accepted. - * The empty password is not supported. - * \param pwdlen Size of the password in bytes. - * Ignored if \p pwd is \c NULL. - * \param f_rng RNG function, must not be \c NULL. Used for blinding. - * \param p_rng RNG parameter - * - * \note On entry, ctx must be empty, either freshly initialised - * with mbedtls_pk_init() or reset with mbedtls_pk_free(). If you need a - * specific key type, check the result with mbedtls_pk_can_do(). - * - * \note The key is also checked for correctness. - * - * \return 0 if successful, or a specific PK or PEM error code - */ -int mbedtls_pk_parse_key(mbedtls_pk_context *ctx, - const unsigned char *key, size_t keylen, - const unsigned char *pwd, size_t pwdlen, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** \ingroup pk_module */ -/** - * \brief Parse a public key in PEM or DER format - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param ctx The PK context to fill. It must have been initialized - * but not set up. - * \param key Input buffer to parse. - * The buffer must contain the input exactly, with no - * extra trailing material. For PEM, the buffer must - * contain a null-terminated string. - * \param keylen Size of \b key in bytes. - * For PEM data, this includes the terminating null byte, - * so \p keylen must be equal to `strlen(key) + 1`. - * - * \note On entry, ctx must be empty, either freshly initialised - * with mbedtls_pk_init() or reset with mbedtls_pk_free(). If you need a - * specific key type, check the result with mbedtls_pk_can_do(). - * - * \note For compressed points, see #MBEDTLS_ECP_PF_COMPRESSED for - * limitations. - * - * \note The key is also checked for correctness. - * - * \return 0 if successful, or a specific PK or PEM error code - */ -int mbedtls_pk_parse_public_key(mbedtls_pk_context *ctx, - const unsigned char *key, size_t keylen); - -#if defined(MBEDTLS_FS_IO) -/** \ingroup pk_module */ -/** - * \brief Load and parse a private key - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param ctx The PK context to fill. It must have been initialized - * but not set up. - * \param path filename to read the private key from - * \param password Optional password to decrypt the file. - * Pass \c NULL if expecting a non-encrypted key. - * Pass a null-terminated string if expecting an encrypted - * key; a non-encrypted key will also be accepted. - * The empty password is not supported. - * \param f_rng RNG function, must not be \c NULL. Used for blinding. - * \param p_rng RNG parameter - * - * \note On entry, ctx must be empty, either freshly initialised - * with mbedtls_pk_init() or reset with mbedtls_pk_free(). If you need a - * specific key type, check the result with mbedtls_pk_can_do(). - * - * \note The key is also checked for correctness. - * - * \return 0 if successful, or a specific PK or PEM error code - */ -int mbedtls_pk_parse_keyfile(mbedtls_pk_context *ctx, - const char *path, const char *password, - mbedtls_f_rng_t *f_rng, void *p_rng); - -/** \ingroup pk_module */ -/** - * \brief Load and parse a public key - * - * \param ctx The PK context to fill. It must have been initialized - * but not set up. - * \param path filename to read the public key from - * - * \note On entry, ctx must be empty, either freshly initialised - * with mbedtls_pk_init() or reset with mbedtls_pk_free(). If - * you need a specific key type, check the result with - * mbedtls_pk_can_do(). - * - * \note The key is also checked for correctness. - * - * \return 0 if successful, or a specific PK or PEM error code - */ -int mbedtls_pk_parse_public_keyfile(mbedtls_pk_context *ctx, const char *path); -#endif /* MBEDTLS_FS_IO */ -#endif /* MBEDTLS_PK_PARSE_C */ - -#if defined(MBEDTLS_PK_WRITE_C) -/** - * \brief Write a private key to a PKCS#1 or SEC1 DER structure - * Note: data is written at the end of the buffer! Use the - * return value to determine where you should start - * using the buffer - * - * \param ctx PK context which must contain a valid private key. - * \param buf buffer to write to - * \param size size of the buffer - * - * \return length of data written if successful, or a specific - * error code - */ -int mbedtls_pk_write_key_der(const mbedtls_pk_context *ctx, unsigned char *buf, size_t size); - -/** - * \brief Write a public key to a SubjectPublicKeyInfo DER structure - * Note: data is written at the end of the buffer! Use the - * return value to determine where you should start - * using the buffer - * - * \param ctx PK context which must contain a valid public or private key. - * \param buf buffer to write to - * \param size size of the buffer - * - * \return length of data written if successful, or a specific - * error code - */ -int mbedtls_pk_write_pubkey_der(const mbedtls_pk_context *ctx, unsigned char *buf, size_t size); - -#if defined(MBEDTLS_PEM_WRITE_C) -/** - * \brief Write a public key to a PEM string - * - * \param ctx PK context which must contain a valid public or private key. - * \param buf Buffer to write to. The output includes a - * terminating null byte. - * \param size Size of the buffer in bytes. - * - * \return 0 if successful, or a specific error code - */ -int mbedtls_pk_write_pubkey_pem(const mbedtls_pk_context *ctx, unsigned char *buf, size_t size); - -/** - * \brief Write a private key to a PKCS#1 or SEC1 PEM string - * - * \param ctx PK context which must contain a valid private key. - * \param buf Buffer to write to. The output includes a - * terminating null byte. - * \param size Size of the buffer in bytes. - * - * \return 0 if successful, or a specific error code - */ -int mbedtls_pk_write_key_pem(const mbedtls_pk_context *ctx, unsigned char *buf, size_t size); -#endif /* MBEDTLS_PEM_WRITE_C */ -#endif /* MBEDTLS_PK_WRITE_C */ - -/* - * WARNING: Low-level functions. You probably do not want to use these unless - * you are certain you do ;) - */ - -#if defined(MBEDTLS_PK_PARSE_C) -/** - * \brief Parse a SubjectPublicKeyInfo DER structure - * - * \param p the position in the ASN.1 data - * \param end end of the buffer - * \param pk The PK context to fill. It must have been initialized - * but not set up. - * - * \return 0 if successful, or a specific PK error code - */ -int mbedtls_pk_parse_subpubkey(unsigned char **p, const unsigned char *end, - mbedtls_pk_context *pk); -#endif /* MBEDTLS_PK_PARSE_C */ - -#if defined(MBEDTLS_PK_WRITE_C) -/** - * \brief Write a subjectPublicKey to ASN.1 data - * Note: function works backwards in data buffer - * - * \param p reference to current position pointer - * \param start start of the buffer (for bounds-checking) - * \param key PK context which must contain a valid public or private key. - * - * \return the length written or a negative error code - */ -int mbedtls_pk_write_pubkey(unsigned char **p, unsigned char *start, - const mbedtls_pk_context *key); -#endif /* MBEDTLS_PK_WRITE_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_PK_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs12.h b/android/mbedtls/arm64-v8a/include/mbedtls/pkcs12.h deleted file mode 100644 index 87f7681f2..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs12.h +++ /dev/null @@ -1,186 +0,0 @@ -/** - * \file pkcs12.h - * - * \brief PKCS#12 Personal Information Exchange Syntax - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PKCS12_H -#define MBEDTLS_PKCS12_H - -#include "mbedtls/build_info.h" - -#include "mbedtls/md.h" -#include "mbedtls/cipher.h" -#include "mbedtls/asn1.h" - -#include - -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_PKCS12_BAD_INPUT_DATA -0x1F80 -/** Feature not available, e.g. unsupported encryption scheme. */ -#define MBEDTLS_ERR_PKCS12_FEATURE_UNAVAILABLE -0x1F00 -/** PBE ASN.1 data not as expected. */ -#define MBEDTLS_ERR_PKCS12_PBE_INVALID_FORMAT -0x1E80 -/** Given private key password does not allow for correct decryption. */ -#define MBEDTLS_ERR_PKCS12_PASSWORD_MISMATCH -0x1E00 - -#define MBEDTLS_PKCS12_DERIVE_KEY 1 /**< encryption/decryption key */ -#define MBEDTLS_PKCS12_DERIVE_IV 2 /**< initialization vector */ -#define MBEDTLS_PKCS12_DERIVE_MAC_KEY 3 /**< integrity / MAC key */ - -#define MBEDTLS_PKCS12_PBE_DECRYPT MBEDTLS_DECRYPT -#define MBEDTLS_PKCS12_PBE_ENCRYPT MBEDTLS_ENCRYPT - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(MBEDTLS_ASN1_PARSE_C) && defined(MBEDTLS_CIPHER_C) - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief PKCS12 Password Based function (encryption / decryption) - * for cipher-based and mbedtls_md-based PBE's - * - * \note When encrypting, #MBEDTLS_CIPHER_PADDING_PKCS7 must - * be enabled at compile time. - * - * \deprecated This function is deprecated and will be removed in a - * future version of the library. - * Please use mbedtls_pkcs12_pbe_ext() instead. - * - * \warning When decrypting: - * - if #MBEDTLS_CIPHER_PADDING_PKCS7 is enabled at compile - * time, this function validates the CBC padding and returns - * #MBEDTLS_ERR_PKCS12_PASSWORD_MISMATCH if the padding is - * invalid. Note that this can help active adversaries - * attempting to brute-forcing the password. Note also that - * there is no guarantee that an invalid password will be - * detected (the chances of a valid padding with a random - * password are about 1/255). - * - if #MBEDTLS_CIPHER_PADDING_PKCS7 is disabled at compile - * time, this function does not validate the CBC padding. - * - * \param pbe_params an ASN1 buffer containing the pkcs-12 PbeParams structure - * \param mode either #MBEDTLS_PKCS12_PBE_ENCRYPT or - * #MBEDTLS_PKCS12_PBE_DECRYPT - * \param cipher_type the cipher used - * \param md_type the mbedtls_md used - * \param pwd Latin1-encoded password used. This may only be \c NULL when - * \p pwdlen is 0. No null terminator should be used. - * \param pwdlen length of the password (may be 0) - * \param data the input data - * \param len data length - * \param output Output buffer. - * On success, it contains the encrypted or decrypted data, - * possibly followed by the CBC padding. - * On failure, the content is indeterminate. - * For decryption, there must be enough room for \p len - * bytes. - * For encryption, there must be enough room for - * \p len + 1 bytes, rounded up to the block size of - * the block cipher identified by \p pbe_params. - * - * \return 0 if successful, or a MBEDTLS_ERR_XXX code - */ -int MBEDTLS_DEPRECATED mbedtls_pkcs12_pbe(mbedtls_asn1_buf *pbe_params, int mode, - mbedtls_cipher_type_t cipher_type, - mbedtls_md_type_t md_type, - const unsigned char *pwd, size_t pwdlen, - const unsigned char *data, size_t len, - unsigned char *output); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -#if defined(MBEDTLS_CIPHER_PADDING_PKCS7) - -/** - * \brief PKCS12 Password Based function (encryption / decryption) - * for cipher-based and mbedtls_md-based PBE's - * - * - * \warning When decrypting: - * - This function validates the CBC padding and returns - * #MBEDTLS_ERR_PKCS12_PASSWORD_MISMATCH if the padding is - * invalid. Note that this can help active adversaries - * attempting to brute-forcing the password. Note also that - * there is no guarantee that an invalid password will be - * detected (the chances of a valid padding with a random - * password are about 1/255). - * - * \param pbe_params an ASN1 buffer containing the pkcs-12 PbeParams structure - * \param mode either #MBEDTLS_PKCS12_PBE_ENCRYPT or - * #MBEDTLS_PKCS12_PBE_DECRYPT - * \param cipher_type the cipher used - * \param md_type the mbedtls_md used - * \param pwd Latin1-encoded password used. This may only be \c NULL when - * \p pwdlen is 0. No null terminator should be used. - * \param pwdlen length of the password (may be 0) - * \param data the input data - * \param len data length - * \param output Output buffer. - * On success, it contains the encrypted or decrypted data, - * possibly followed by the CBC padding. - * On failure, the content is indeterminate. - * For decryption, there must be enough room for \p len - * bytes. - * For encryption, there must be enough room for - * \p len + 1 bytes, rounded up to the block size of - * the block cipher identified by \p pbe_params. - * \param output_size size of output buffer. - * This must be big enough to accommodate for output plus - * padding data. - * \param output_len On success, length of actual data written to the output buffer. - * - * \return 0 if successful, or a MBEDTLS_ERR_XXX code - */ -int mbedtls_pkcs12_pbe_ext(mbedtls_asn1_buf *pbe_params, int mode, - mbedtls_cipher_type_t cipher_type, mbedtls_md_type_t md_type, - const unsigned char *pwd, size_t pwdlen, - const unsigned char *data, size_t len, - unsigned char *output, size_t output_size, - size_t *output_len); - -#endif /* MBEDTLS_CIPHER_PADDING_PKCS7 */ - -#endif /* MBEDTLS_ASN1_PARSE_C && MBEDTLS_CIPHER_C */ - -/** - * \brief The PKCS#12 derivation function uses a password and a salt - * to produce pseudo-random bits for a particular "purpose". - * - * Depending on the given id, this function can produce an - * encryption/decryption key, an initialization vector or an - * integrity key. - * - * \param data buffer to store the derived data in - * \param datalen length of buffer to fill - * \param pwd The password to use. For compliance with PKCS#12 §B.1, this - * should be a BMPString, i.e. a Unicode string where each - * character is encoded as 2 bytes in big-endian order, with - * no byte order mark and with a null terminator (i.e. the - * last two bytes should be 0x00 0x00). - * \param pwdlen length of the password (may be 0). - * \param salt Salt buffer to use. This may only be \c NULL when - * \p saltlen is 0. - * \param saltlen length of the salt (may be zero) - * \param mbedtls_md mbedtls_md type to use during the derivation - * \param id id that describes the purpose (can be - * #MBEDTLS_PKCS12_DERIVE_KEY, #MBEDTLS_PKCS12_DERIVE_IV or - * #MBEDTLS_PKCS12_DERIVE_MAC_KEY) - * \param iterations number of iterations - * - * \return 0 if successful, or a MD, BIGNUM type error. - */ -int mbedtls_pkcs12_derivation(unsigned char *data, size_t datalen, - const unsigned char *pwd, size_t pwdlen, - const unsigned char *salt, size_t saltlen, - mbedtls_md_type_t mbedtls_md, int id, int iterations); - -#ifdef __cplusplus -} -#endif - -#endif /* pkcs12.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs5.h b/android/mbedtls/arm64-v8a/include/mbedtls/pkcs5.h deleted file mode 100644 index 9ba5689d4..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs5.h +++ /dev/null @@ -1,198 +0,0 @@ -/** - * \file pkcs5.h - * - * \brief PKCS#5 functions - * - * \author Mathias Olsson - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PKCS5_H -#define MBEDTLS_PKCS5_H - -#include "mbedtls/build_info.h" -#include "mbedtls/platform_util.h" - -#include "mbedtls/asn1.h" -#include "mbedtls/md.h" -#include "mbedtls/cipher.h" - -#include -#include - -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_PKCS5_BAD_INPUT_DATA -0x2f80 -/** Unexpected ASN.1 data. */ -#define MBEDTLS_ERR_PKCS5_INVALID_FORMAT -0x2f00 -/** Requested encryption or digest alg not available. */ -#define MBEDTLS_ERR_PKCS5_FEATURE_UNAVAILABLE -0x2e80 -/** Given private key password does not allow for correct decryption. */ -#define MBEDTLS_ERR_PKCS5_PASSWORD_MISMATCH -0x2e00 - -#define MBEDTLS_PKCS5_DECRYPT MBEDTLS_DECRYPT -#define MBEDTLS_PKCS5_ENCRYPT MBEDTLS_ENCRYPT - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(MBEDTLS_ASN1_PARSE_C) && defined(MBEDTLS_CIPHER_C) - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief PKCS#5 PBES2 function - * - * \note When encrypting, #MBEDTLS_CIPHER_PADDING_PKCS7 must - * be enabled at compile time. - * - * \deprecated This function is deprecated and will be removed in a - * future version of the library. - * Please use mbedtls_pkcs5_pbes2_ext() instead. - * - * \warning When decrypting: - * - if #MBEDTLS_CIPHER_PADDING_PKCS7 is enabled at compile - * time, this function validates the CBC padding and returns - * #MBEDTLS_ERR_PKCS5_PASSWORD_MISMATCH if the padding is - * invalid. Note that this can help active adversaries - * attempting to brute-forcing the password. Note also that - * there is no guarantee that an invalid password will be - * detected (the chances of a valid padding with a random - * password are about 1/255). - * - if #MBEDTLS_CIPHER_PADDING_PKCS7 is disabled at compile - * time, this function does not validate the CBC padding. - * - * \param pbe_params the ASN.1 algorithm parameters - * \param mode either #MBEDTLS_PKCS5_DECRYPT or #MBEDTLS_PKCS5_ENCRYPT - * \param pwd password to use when generating key - * \param pwdlen length of password - * \param data data to process - * \param datalen length of data - * \param output Output buffer. - * On success, it contains the encrypted or decrypted data, - * possibly followed by the CBC padding. - * On failure, the content is indeterminate. - * For decryption, there must be enough room for \p datalen - * bytes. - * For encryption, there must be enough room for - * \p datalen + 1 bytes, rounded up to the block size of - * the block cipher identified by \p pbe_params. - * - * \returns 0 on success, or a MBEDTLS_ERR_XXX code if verification fails. - */ -int MBEDTLS_DEPRECATED mbedtls_pkcs5_pbes2(const mbedtls_asn1_buf *pbe_params, int mode, - const unsigned char *pwd, size_t pwdlen, - const unsigned char *data, size_t datalen, - unsigned char *output); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -#if defined(MBEDTLS_CIPHER_PADDING_PKCS7) - -/** - * \brief PKCS#5 PBES2 function - * - * \warning When decrypting: - * - This function validates the CBC padding and returns - * #MBEDTLS_ERR_PKCS5_PASSWORD_MISMATCH if the padding is - * invalid. Note that this can help active adversaries - * attempting to brute-forcing the password. Note also that - * there is no guarantee that an invalid password will be - * detected (the chances of a valid padding with a random - * password are about 1/255). - * - * \param pbe_params the ASN.1 algorithm parameters - * \param mode either #MBEDTLS_PKCS5_DECRYPT or #MBEDTLS_PKCS5_ENCRYPT - * \param pwd password to use when generating key - * \param pwdlen length of password - * \param data data to process - * \param datalen length of data - * \param output Output buffer. - * On success, it contains the decrypted data. - * On failure, the content is indetermidate. - * For decryption, there must be enough room for \p datalen - * bytes. - * For encryption, there must be enough room for - * \p datalen + 1 bytes, rounded up to the block size of - * the block cipher identified by \p pbe_params. - * \param output_size size of output buffer. - * This must be big enough to accommodate for output plus - * padding data. - * \param output_len On success, length of actual data written to the output buffer. - * - * \returns 0 on success, or a MBEDTLS_ERR_XXX code if parsing or decryption fails. - */ -int mbedtls_pkcs5_pbes2_ext(const mbedtls_asn1_buf *pbe_params, int mode, - const unsigned char *pwd, size_t pwdlen, - const unsigned char *data, size_t datalen, - unsigned char *output, size_t output_size, - size_t *output_len); - -#endif /* MBEDTLS_CIPHER_PADDING_PKCS7 */ - -#endif /* MBEDTLS_ASN1_PARSE_C && MBEDTLS_CIPHER_C*/ - -/** - * \brief PKCS#5 PBKDF2 using HMAC without using the HMAC context - * - * \param md_type Hash algorithm used - * \param password Password to use when generating key - * \param plen Length of password - * \param salt Salt to use when generating key - * \param slen Length of salt - * \param iteration_count Iteration count - * \param key_length Length of generated key in bytes - * \param output Generated key. Must be at least as big as key_length - * - * \returns 0 on success, or a MBEDTLS_ERR_XXX code if verification fails. - */ -int mbedtls_pkcs5_pbkdf2_hmac_ext(mbedtls_md_type_t md_type, - const unsigned char *password, - size_t plen, const unsigned char *salt, size_t slen, - unsigned int iteration_count, - uint32_t key_length, unsigned char *output); - -#if defined(MBEDTLS_MD_C) -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief PKCS#5 PBKDF2 using HMAC - * - * \deprecated Superseded by mbedtls_pkcs5_pbkdf2_hmac_ext(). - * - * \param ctx Generic HMAC context - * \param password Password to use when generating key - * \param plen Length of password - * \param salt Salt to use when generating key - * \param slen Length of salt - * \param iteration_count Iteration count - * \param key_length Length of generated key in bytes - * \param output Generated key. Must be at least as big as key_length - * - * \returns 0 on success, or a MBEDTLS_ERR_XXX code if verification fails. - */ -int MBEDTLS_DEPRECATED mbedtls_pkcs5_pbkdf2_hmac(mbedtls_md_context_t *ctx, - const unsigned char *password, - size_t plen, - const unsigned char *salt, - size_t slen, - unsigned int iteration_count, - uint32_t key_length, - unsigned char *output); -#endif /* !MBEDTLS_DEPRECATED_REMOVED */ -#endif /* MBEDTLS_MD_C */ -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -int mbedtls_pkcs5_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* pkcs5.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs7.h b/android/mbedtls/arm64-v8a/include/mbedtls/pkcs7.h deleted file mode 100644 index e9b482208..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/pkcs7.h +++ /dev/null @@ -1,240 +0,0 @@ -/** - * \file pkcs7.h - * - * \brief PKCS #7 generic defines and structures - * https://tools.ietf.org/html/rfc2315 - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -/** - * Note: For the time being, this implementation of the PKCS #7 cryptographic - * message syntax is a partial implementation of RFC 2315. - * Differences include: - * - The RFC specifies 6 different content types. The only type currently - * supported in Mbed TLS is the signed-data content type. - * - The only supported PKCS #7 Signed Data syntax version is version 1 - * - The RFC specifies support for BER. This implementation is limited to - * DER only. - * - The RFC specifies that multiple digest algorithms can be specified - * in the Signed Data type. Only one digest algorithm is supported in Mbed TLS. - * - The RFC specifies the Signed Data type can contain multiple X.509 or PKCS #6 extended - * certificates. In Mbed TLS, this list can only contain 0 or 1 certificates - * and they must be in X.509 format. - * - The RFC specifies the Signed Data type can contain - * certificate-revocation lists (CRLs). This implementation has no support - * for CRLs so it is assumed to be an empty list. - * - The RFC allows for SignerInfo structure to optionally contain - * unauthenticatedAttributes and authenticatedAttributes. In Mbed TLS it is - * assumed these fields are empty. - * - The RFC allows for the signed Data type to contain contentInfo. This - * implementation assumes the type is DATA and the content is empty. - */ - -#ifndef MBEDTLS_PKCS7_H -#define MBEDTLS_PKCS7_H - -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/asn1.h" -#include "mbedtls/x509_crt.h" - -/** - * \name PKCS #7 Module Error codes - * \{ - */ -#define MBEDTLS_ERR_PKCS7_INVALID_FORMAT -0x5300 /**< The format is invalid, e.g. different type expected. */ -#define MBEDTLS_ERR_PKCS7_FEATURE_UNAVAILABLE -0x5380 /**< Unavailable feature, e.g. anything other than signed data. */ -#define MBEDTLS_ERR_PKCS7_INVALID_VERSION -0x5400 /**< The PKCS #7 version element is invalid or cannot be parsed. */ -#define MBEDTLS_ERR_PKCS7_INVALID_CONTENT_INFO -0x5480 /**< The PKCS #7 content info is invalid or cannot be parsed. */ -#define MBEDTLS_ERR_PKCS7_INVALID_ALG -0x5500 /**< The algorithm tag or value is invalid or cannot be parsed. */ -#define MBEDTLS_ERR_PKCS7_INVALID_CERT -0x5580 /**< The certificate tag or value is invalid or cannot be parsed. */ -#define MBEDTLS_ERR_PKCS7_INVALID_SIGNATURE -0x5600 /**< Error parsing the signature */ -#define MBEDTLS_ERR_PKCS7_INVALID_SIGNER_INFO -0x5680 /**< Error parsing the signer's info */ -#define MBEDTLS_ERR_PKCS7_BAD_INPUT_DATA -0x5700 /**< Input invalid. */ -#define MBEDTLS_ERR_PKCS7_ALLOC_FAILED -0x5780 /**< Allocation of memory failed. */ -#define MBEDTLS_ERR_PKCS7_VERIFY_FAIL -0x5800 /**< Verification Failed */ -#define MBEDTLS_ERR_PKCS7_CERT_DATE_INVALID -0x5880 /**< The PKCS #7 date issued/expired dates are invalid */ -/* \} name */ - -/** - * \name PKCS #7 Supported Version - * \{ - */ -#define MBEDTLS_PKCS7_SUPPORTED_VERSION 0x01 -/* \} name */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Type-length-value structure that allows for ASN.1 using DER. - */ -typedef mbedtls_asn1_buf mbedtls_pkcs7_buf; - -/** - * Container for ASN.1 named information objects. - * It allows for Relative Distinguished Names (e.g. cn=localhost,ou=code,etc.). - */ -typedef mbedtls_asn1_named_data mbedtls_pkcs7_name; - -/** - * Container for a sequence of ASN.1 items - */ -typedef mbedtls_asn1_sequence mbedtls_pkcs7_sequence; - -/** - * PKCS #7 types - */ -typedef enum { - MBEDTLS_PKCS7_NONE=0, - MBEDTLS_PKCS7_DATA, - MBEDTLS_PKCS7_SIGNED_DATA, - MBEDTLS_PKCS7_ENVELOPED_DATA, - MBEDTLS_PKCS7_SIGNED_AND_ENVELOPED_DATA, - MBEDTLS_PKCS7_DIGESTED_DATA, - MBEDTLS_PKCS7_ENCRYPTED_DATA, -} -mbedtls_pkcs7_type; - -/** - * Structure holding PKCS #7 signer info - */ -typedef struct mbedtls_pkcs7_signer_info { - int MBEDTLS_PRIVATE(version); - mbedtls_x509_buf MBEDTLS_PRIVATE(serial); - mbedtls_x509_name MBEDTLS_PRIVATE(issuer); - mbedtls_x509_buf MBEDTLS_PRIVATE(issuer_raw); - mbedtls_x509_buf MBEDTLS_PRIVATE(alg_identifier); - mbedtls_x509_buf MBEDTLS_PRIVATE(sig_alg_identifier); - mbedtls_x509_buf MBEDTLS_PRIVATE(sig); - struct mbedtls_pkcs7_signer_info *MBEDTLS_PRIVATE(next); -} -mbedtls_pkcs7_signer_info; - -/** - * Structure holding the signed data section - */ -typedef struct mbedtls_pkcs7_signed_data { - int MBEDTLS_PRIVATE(version); - mbedtls_pkcs7_buf MBEDTLS_PRIVATE(digest_alg_identifiers); - int MBEDTLS_PRIVATE(no_of_certs); - mbedtls_x509_crt MBEDTLS_PRIVATE(certs); - int MBEDTLS_PRIVATE(no_of_crls); - mbedtls_x509_crl MBEDTLS_PRIVATE(crl); - int MBEDTLS_PRIVATE(no_of_signers); - mbedtls_pkcs7_signer_info MBEDTLS_PRIVATE(signers); -} -mbedtls_pkcs7_signed_data; - -/** - * Structure holding PKCS #7 structure, only signed data for now - */ -typedef struct mbedtls_pkcs7 { - mbedtls_pkcs7_buf MBEDTLS_PRIVATE(raw); - mbedtls_pkcs7_signed_data MBEDTLS_PRIVATE(signed_data); -} -mbedtls_pkcs7; - -/** - * \brief Initialize mbedtls_pkcs7 structure. - * - * \param pkcs7 mbedtls_pkcs7 structure. - */ -void mbedtls_pkcs7_init(mbedtls_pkcs7 *pkcs7); - -/** - * \brief Parse a single DER formatted PKCS #7 detached signature. - * - * \param pkcs7 The mbedtls_pkcs7 structure to be filled by the parser. - * \param buf The buffer holding only the DER encoded PKCS #7 content. - * \param buflen The size in bytes of \p buf. The size must be exactly the - * length of the DER encoded PKCS #7 content. - * - * \note This function makes an internal copy of the PKCS #7 buffer - * \p buf. In particular, \p buf may be destroyed or reused - * after this call returns. - * \note Signatures with internal data are not supported. - * - * \return The \c mbedtls_pkcs7_type of \p buf, if successful. - * \return A negative error code on failure. - */ -int mbedtls_pkcs7_parse_der(mbedtls_pkcs7 *pkcs7, const unsigned char *buf, - const size_t buflen); - -/** - * \brief Verification of PKCS #7 signature against a caller-supplied - * certificate. - * - * For each signer in the PKCS structure, this function computes - * a signature over the supplied data, using the supplied - * certificate and the same digest algorithm as specified by the - * signer. It then compares this signature against the - * signer's signature; verification succeeds if any comparison - * matches. - * - * This function does not use the certificates held within the - * PKCS #7 structure itself, and does not check that the - * certificate is signed by a trusted certification authority. - * - * \param pkcs7 mbedtls_pkcs7 structure containing signature. - * \param cert Certificate containing key to verify signature. - * \param data Plain data on which signature has to be verified. - * \param datalen Length of the data. - * - * \note This function internally calculates the hash on the supplied - * plain data for signature verification. - * - * \return 0 if the signature verifies, or a negative error code on failure. - */ -int mbedtls_pkcs7_signed_data_verify(mbedtls_pkcs7 *pkcs7, - const mbedtls_x509_crt *cert, - const unsigned char *data, - size_t datalen); - -/** - * \brief Verification of PKCS #7 signature against a caller-supplied - * certificate. - * - * For each signer in the PKCS structure, this function - * validates a signature over the supplied hash, using the - * supplied certificate and the same digest algorithm as - * specified by the signer. Verification succeeds if any - * signature is good. - * - * This function does not use the certificates held within the - * PKCS #7 structure itself, and does not check that the - * certificate is signed by a trusted certification authority. - * - * \param pkcs7 PKCS #7 structure containing signature. - * \param cert Certificate containing key to verify signature. - * \param hash Hash of the plain data on which signature has to be verified. - * \param hashlen Length of the hash. - * - * \note This function is different from mbedtls_pkcs7_signed_data_verify() - * in that it is directly passed the hash of the data. - * - * \return 0 if the signature verifies, or a negative error code on failure. - */ -int mbedtls_pkcs7_signed_hash_verify(mbedtls_pkcs7 *pkcs7, - const mbedtls_x509_crt *cert, - const unsigned char *hash, size_t hashlen); - -/** - * \brief Unallocate all PKCS #7 data and zeroize the memory. - * It doesn't free \p pkcs7 itself. This should be done by the caller. - * - * \param pkcs7 mbedtls_pkcs7 structure to free. - */ -void mbedtls_pkcs7_free(mbedtls_pkcs7 *pkcs7); - -#ifdef __cplusplus -} -#endif - -#endif /* pkcs7.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/platform.h b/android/mbedtls/arm64-v8a/include/mbedtls/platform.h deleted file mode 100644 index de3d71d9d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/platform.h +++ /dev/null @@ -1,485 +0,0 @@ -/** - * \file platform.h - * - * \brief This file contains the definitions and functions of the - * Mbed TLS platform abstraction layer. - * - * The platform abstraction layer removes the need for the library - * to directly link to standard C library functions or operating - * system services, making the library easier to port and embed. - * Application developers and users of the library can provide their own - * implementations of these functions, or implementations specific to - * their platform, which can be statically linked to the library or - * dynamically configured at runtime. - * - * When all compilation options related to platform abstraction are - * disabled, this header just defines `mbedtls_xxx` function names - * as aliases to the standard `xxx` function. - * - * Most modules in the library and example programs are expected to - * include this header. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PLATFORM_H -#define MBEDTLS_PLATFORM_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#if defined(MBEDTLS_HAVE_TIME) -#include "mbedtls/platform_time.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -/* The older Microsoft Windows common runtime provides non-conforming - * implementations of some standard library functions, including snprintf - * and vsnprintf. This affects MSVC and MinGW builds. - */ -#if defined(__MINGW32__) || (defined(_MSC_VER) && _MSC_VER <= 1900) -#define MBEDTLS_PLATFORM_HAS_NON_CONFORMING_SNPRINTF -#define MBEDTLS_PLATFORM_HAS_NON_CONFORMING_VSNPRINTF -#endif - -#if !defined(MBEDTLS_PLATFORM_NO_STD_FUNCTIONS) -#include -#include -#if defined(MBEDTLS_HAVE_TIME) -#include -#endif -#if !defined(MBEDTLS_PLATFORM_STD_SNPRINTF) -#if defined(MBEDTLS_PLATFORM_HAS_NON_CONFORMING_SNPRINTF) -#define MBEDTLS_PLATFORM_STD_SNPRINTF mbedtls_platform_win32_snprintf /**< The default \c snprintf function to use. */ -#else -#define MBEDTLS_PLATFORM_STD_SNPRINTF snprintf /**< The default \c snprintf function to use. */ -#endif -#endif -#if !defined(MBEDTLS_PLATFORM_STD_VSNPRINTF) -#if defined(MBEDTLS_PLATFORM_HAS_NON_CONFORMING_VSNPRINTF) -#define MBEDTLS_PLATFORM_STD_VSNPRINTF mbedtls_platform_win32_vsnprintf /**< The default \c vsnprintf function to use. */ -#else -#define MBEDTLS_PLATFORM_STD_VSNPRINTF vsnprintf /**< The default \c vsnprintf function to use. */ -#endif -#endif -#if !defined(MBEDTLS_PLATFORM_STD_PRINTF) -#define MBEDTLS_PLATFORM_STD_PRINTF printf /**< The default \c printf function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_FPRINTF) -#define MBEDTLS_PLATFORM_STD_FPRINTF fprintf /**< The default \c fprintf function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_CALLOC) -#define MBEDTLS_PLATFORM_STD_CALLOC calloc /**< The default \c calloc function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_FREE) -#define MBEDTLS_PLATFORM_STD_FREE free /**< The default \c free function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_SETBUF) -#define MBEDTLS_PLATFORM_STD_SETBUF setbuf /**< The default \c setbuf function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_EXIT) -#define MBEDTLS_PLATFORM_STD_EXIT exit /**< The default \c exit function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_TIME) -#define MBEDTLS_PLATFORM_STD_TIME time /**< The default \c time function to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_EXIT_SUCCESS) -#define MBEDTLS_PLATFORM_STD_EXIT_SUCCESS EXIT_SUCCESS /**< The default exit value to use. */ -#endif -#if !defined(MBEDTLS_PLATFORM_STD_EXIT_FAILURE) -#define MBEDTLS_PLATFORM_STD_EXIT_FAILURE EXIT_FAILURE /**< The default exit value to use. */ -#endif -#if defined(MBEDTLS_FS_IO) -#if !defined(MBEDTLS_PLATFORM_STD_NV_SEED_READ) -#define MBEDTLS_PLATFORM_STD_NV_SEED_READ mbedtls_platform_std_nv_seed_read -#endif -#if !defined(MBEDTLS_PLATFORM_STD_NV_SEED_WRITE) -#define MBEDTLS_PLATFORM_STD_NV_SEED_WRITE mbedtls_platform_std_nv_seed_write -#endif -#if !defined(MBEDTLS_PLATFORM_STD_NV_SEED_FILE) -#define MBEDTLS_PLATFORM_STD_NV_SEED_FILE "seedfile" -#endif -#endif /* MBEDTLS_FS_IO */ -#else /* MBEDTLS_PLATFORM_NO_STD_FUNCTIONS */ -#if defined(MBEDTLS_PLATFORM_STD_MEM_HDR) -#include MBEDTLS_PLATFORM_STD_MEM_HDR -#endif -#endif /* MBEDTLS_PLATFORM_NO_STD_FUNCTIONS */ - -/* Enable certain documented defines only when generating doxygen to avoid - * an "unrecognized define" error. */ -#if defined(__DOXYGEN__) && !defined(MBEDTLS_PLATFORM_STD_CALLOC) -#define MBEDTLS_PLATFORM_STD_CALLOC -#endif - -#if defined(__DOXYGEN__) && !defined(MBEDTLS_PLATFORM_STD_FREE) -#define MBEDTLS_PLATFORM_STD_FREE -#endif - -/** \} name SECTION: Module settings */ - -/* - * The function pointers for calloc and free. - * Please see MBEDTLS_PLATFORM_STD_CALLOC and MBEDTLS_PLATFORM_STD_FREE - * in mbedtls_config.h for more information about behaviour and requirements. - */ -#if defined(MBEDTLS_PLATFORM_MEMORY) -#if defined(MBEDTLS_PLATFORM_FREE_MACRO) && \ - defined(MBEDTLS_PLATFORM_CALLOC_MACRO) -#undef mbedtls_free -#undef mbedtls_calloc -#define mbedtls_free MBEDTLS_PLATFORM_FREE_MACRO -#define mbedtls_calloc MBEDTLS_PLATFORM_CALLOC_MACRO -#else -/* For size_t */ -#include -extern void *mbedtls_calloc(size_t n, size_t size); -extern void mbedtls_free(void *ptr); - -/** - * \brief This function dynamically sets the memory-management - * functions used by the library, during runtime. - * - * \param calloc_func The \c calloc function implementation. - * \param free_func The \c free function implementation. - * - * \return \c 0. - */ -int mbedtls_platform_set_calloc_free(void *(*calloc_func)(size_t, size_t), - void (*free_func)(void *)); -#endif /* MBEDTLS_PLATFORM_FREE_MACRO && MBEDTLS_PLATFORM_CALLOC_MACRO */ -#else /* !MBEDTLS_PLATFORM_MEMORY */ -#undef mbedtls_free -#undef mbedtls_calloc -#define mbedtls_free free -#define mbedtls_calloc calloc -#endif /* MBEDTLS_PLATFORM_MEMORY && !MBEDTLS_PLATFORM_{FREE,CALLOC}_MACRO */ - -/* - * The function pointers for fprintf - */ -#if defined(MBEDTLS_PLATFORM_FPRINTF_ALT) -/* We need FILE * */ -#include -extern int (*mbedtls_fprintf)(FILE *stream, const char *format, ...); - -/** - * \brief This function dynamically configures the fprintf - * function that is called when the - * mbedtls_fprintf() function is invoked by the library. - * - * \param fprintf_func The \c fprintf function implementation. - * - * \return \c 0. - */ -int mbedtls_platform_set_fprintf(int (*fprintf_func)(FILE *stream, const char *, - ...)); -#else -#undef mbedtls_fprintf -#if defined(MBEDTLS_PLATFORM_FPRINTF_MACRO) -#define mbedtls_fprintf MBEDTLS_PLATFORM_FPRINTF_MACRO -#else -#define mbedtls_fprintf fprintf -#endif /* MBEDTLS_PLATFORM_FPRINTF_MACRO */ -#endif /* MBEDTLS_PLATFORM_FPRINTF_ALT */ - -/* - * The function pointers for printf - */ -#if defined(MBEDTLS_PLATFORM_PRINTF_ALT) -extern int (*mbedtls_printf)(const char *format, ...); - -/** - * \brief This function dynamically configures the snprintf - * function that is called when the mbedtls_snprintf() - * function is invoked by the library. - * - * \param printf_func The \c printf function implementation. - * - * \return \c 0 on success. - */ -int mbedtls_platform_set_printf(int (*printf_func)(const char *, ...)); -#else /* !MBEDTLS_PLATFORM_PRINTF_ALT */ -#undef mbedtls_printf -#if defined(MBEDTLS_PLATFORM_PRINTF_MACRO) -#define mbedtls_printf MBEDTLS_PLATFORM_PRINTF_MACRO -#else -#define mbedtls_printf printf -#endif /* MBEDTLS_PLATFORM_PRINTF_MACRO */ -#endif /* MBEDTLS_PLATFORM_PRINTF_ALT */ - -/* - * The function pointers for snprintf - * - * The snprintf implementation should conform to C99: - * - it *must* always correctly zero-terminate the buffer - * (except when n == 0, then it must leave the buffer untouched) - * - however it is acceptable to return -1 instead of the required length when - * the destination buffer is too short. - */ -#if defined(MBEDTLS_PLATFORM_HAS_NON_CONFORMING_SNPRINTF) -/* For Windows (inc. MSYS2), we provide our own fixed implementation */ -int mbedtls_platform_win32_snprintf(char *s, size_t n, const char *fmt, ...); -#endif - -#if defined(MBEDTLS_PLATFORM_SNPRINTF_ALT) -extern int (*mbedtls_snprintf)(char *s, size_t n, const char *format, ...); - -/** - * \brief This function allows configuring a custom - * \c snprintf function pointer. - * - * \param snprintf_func The \c snprintf function implementation. - * - * \return \c 0 on success. - */ -int mbedtls_platform_set_snprintf(int (*snprintf_func)(char *s, size_t n, - const char *format, ...)); -#else /* MBEDTLS_PLATFORM_SNPRINTF_ALT */ -#undef mbedtls_snprintf -#if defined(MBEDTLS_PLATFORM_SNPRINTF_MACRO) -#define mbedtls_snprintf MBEDTLS_PLATFORM_SNPRINTF_MACRO -#else -#define mbedtls_snprintf MBEDTLS_PLATFORM_STD_SNPRINTF -#endif /* MBEDTLS_PLATFORM_SNPRINTF_MACRO */ -#endif /* MBEDTLS_PLATFORM_SNPRINTF_ALT */ - -/* - * The function pointers for vsnprintf - * - * The vsnprintf implementation should conform to C99: - * - it *must* always correctly zero-terminate the buffer - * (except when n == 0, then it must leave the buffer untouched) - * - however it is acceptable to return -1 instead of the required length when - * the destination buffer is too short. - */ -#if defined(MBEDTLS_PLATFORM_HAS_NON_CONFORMING_VSNPRINTF) -#include -/* For Older Windows (inc. MSYS2), we provide our own fixed implementation */ -int mbedtls_platform_win32_vsnprintf(char *s, size_t n, const char *fmt, va_list arg); -#endif - -#if defined(MBEDTLS_PLATFORM_VSNPRINTF_ALT) -#include -extern int (*mbedtls_vsnprintf)(char *s, size_t n, const char *format, va_list arg); - -/** - * \brief Set your own snprintf function pointer - * - * \param vsnprintf_func The \c vsnprintf function implementation - * - * \return \c 0 - */ -int mbedtls_platform_set_vsnprintf(int (*vsnprintf_func)(char *s, size_t n, - const char *format, va_list arg)); -#else /* MBEDTLS_PLATFORM_VSNPRINTF_ALT */ -#undef mbedtls_vsnprintf -#if defined(MBEDTLS_PLATFORM_VSNPRINTF_MACRO) -#define mbedtls_vsnprintf MBEDTLS_PLATFORM_VSNPRINTF_MACRO -#else -#define mbedtls_vsnprintf vsnprintf -#endif /* MBEDTLS_PLATFORM_VSNPRINTF_MACRO */ -#endif /* MBEDTLS_PLATFORM_VSNPRINTF_ALT */ - -/* - * The function pointers for setbuf - */ -#if defined(MBEDTLS_PLATFORM_SETBUF_ALT) -#include -/** - * \brief Function pointer to call for `setbuf()` functionality - * (changing the internal buffering on stdio calls). - * - * \note The library calls this function to disable - * buffering when reading or writing sensitive data, - * to avoid having extra copies of sensitive data - * remaining in stdio buffers after the file is - * closed. If this is not a concern, for example if - * your platform's stdio doesn't have any buffering, - * you can set mbedtls_setbuf to a function that - * does nothing. - * - * The library always calls this function with - * `buf` equal to `NULL`. - */ -extern void (*mbedtls_setbuf)(FILE *stream, char *buf); - -/** - * \brief Dynamically configure the function that is called - * when the mbedtls_setbuf() function is called by the - * library. - * - * \param setbuf_func The \c setbuf function implementation - * - * \return \c 0 - */ -int mbedtls_platform_set_setbuf(void (*setbuf_func)( - FILE *stream, char *buf)); -#else -#undef mbedtls_setbuf -#if defined(MBEDTLS_PLATFORM_SETBUF_MACRO) -/** - * \brief Macro defining the function for the library to - * call for `setbuf` functionality (changing the - * internal buffering on stdio calls). - * - * \note See extra comments on the mbedtls_setbuf() function - * pointer above. - * - * \return \c 0 on success, negative on error. - */ -#define mbedtls_setbuf MBEDTLS_PLATFORM_SETBUF_MACRO -#else -#define mbedtls_setbuf setbuf -#endif /* MBEDTLS_PLATFORM_SETBUF_MACRO */ -#endif /* MBEDTLS_PLATFORM_SETBUF_ALT */ - -/* - * The function pointers for exit - */ -#if defined(MBEDTLS_PLATFORM_EXIT_ALT) -extern void (*mbedtls_exit)(int status); - -/** - * \brief This function dynamically configures the exit - * function that is called when the mbedtls_exit() - * function is invoked by the library. - * - * \param exit_func The \c exit function implementation. - * - * \return \c 0 on success. - */ -int mbedtls_platform_set_exit(void (*exit_func)(int status)); -#else -#undef mbedtls_exit -#if defined(MBEDTLS_PLATFORM_EXIT_MACRO) -#define mbedtls_exit MBEDTLS_PLATFORM_EXIT_MACRO -#else -#define mbedtls_exit exit -#endif /* MBEDTLS_PLATFORM_EXIT_MACRO */ -#endif /* MBEDTLS_PLATFORM_EXIT_ALT */ - -/* - * The default exit values - */ -#if defined(MBEDTLS_PLATFORM_STD_EXIT_SUCCESS) -#define MBEDTLS_EXIT_SUCCESS MBEDTLS_PLATFORM_STD_EXIT_SUCCESS -#else -#define MBEDTLS_EXIT_SUCCESS 0 -#endif -#if defined(MBEDTLS_PLATFORM_STD_EXIT_FAILURE) -#define MBEDTLS_EXIT_FAILURE MBEDTLS_PLATFORM_STD_EXIT_FAILURE -#else -#define MBEDTLS_EXIT_FAILURE 1 -#endif - -/* - * The function pointers for reading from and writing a seed file to - * Non-Volatile storage (NV) in a platform-independent way - * - * Only enabled when the NV seed entropy source is enabled - */ -#if defined(MBEDTLS_ENTROPY_NV_SEED) -#if !defined(MBEDTLS_PLATFORM_NO_STD_FUNCTIONS) && defined(MBEDTLS_FS_IO) -/* Internal standard platform definitions */ -int mbedtls_platform_std_nv_seed_read(unsigned char *buf, size_t buf_len); -int mbedtls_platform_std_nv_seed_write(unsigned char *buf, size_t buf_len); -#endif - -#if defined(MBEDTLS_PLATFORM_NV_SEED_ALT) -extern int (*mbedtls_nv_seed_read)(unsigned char *buf, size_t buf_len); -extern int (*mbedtls_nv_seed_write)(unsigned char *buf, size_t buf_len); - -/** - * \brief This function allows configuring custom seed file writing and - * reading functions. - * - * \param nv_seed_read_func The seed reading function implementation. - * \param nv_seed_write_func The seed writing function implementation. - * - * \return \c 0 on success. - */ -int mbedtls_platform_set_nv_seed( - int (*nv_seed_read_func)(unsigned char *buf, size_t buf_len), - int (*nv_seed_write_func)(unsigned char *buf, size_t buf_len) - ); -#else -#undef mbedtls_nv_seed_read -#undef mbedtls_nv_seed_write -#if defined(MBEDTLS_PLATFORM_NV_SEED_READ_MACRO) && \ - defined(MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO) -#define mbedtls_nv_seed_read MBEDTLS_PLATFORM_NV_SEED_READ_MACRO -#define mbedtls_nv_seed_write MBEDTLS_PLATFORM_NV_SEED_WRITE_MACRO -#else -#define mbedtls_nv_seed_read mbedtls_platform_std_nv_seed_read -#define mbedtls_nv_seed_write mbedtls_platform_std_nv_seed_write -#endif -#endif /* MBEDTLS_PLATFORM_NV_SEED_ALT */ -#endif /* MBEDTLS_ENTROPY_NV_SEED */ - -#if !defined(MBEDTLS_PLATFORM_SETUP_TEARDOWN_ALT) - -/** - * \brief The platform context structure. - * - * \note This structure may be used to assist platform-specific - * setup or teardown operations. - */ -typedef struct mbedtls_platform_context { - char MBEDTLS_PRIVATE(dummy); /**< A placeholder member, as empty structs are not portable. */ -} -mbedtls_platform_context; - -#else -#include "platform_alt.h" -#endif /* !MBEDTLS_PLATFORM_SETUP_TEARDOWN_ALT */ - -/** - * \brief This function performs any platform-specific initialization - * operations. - * - * \note This function should be called before any other library functions. - * - * Its implementation is platform-specific, and unless - * platform-specific code is provided, it does nothing. - * - * \note The usage and necessity of this function is dependent on the platform. - * - * \param ctx The platform context. - * - * \return \c 0 on success. - */ -int mbedtls_platform_setup(mbedtls_platform_context *ctx); -/** - * \brief This function performs any platform teardown operations. - * - * \note This function should be called after every other Mbed TLS module - * has been correctly freed using the appropriate free function. - * - * Its implementation is platform-specific, and unless - * platform-specific code is provided, it does nothing. - * - * \note The usage and necessity of this function is dependent on the platform. - * - * \param ctx The platform context. - * - */ -void mbedtls_platform_teardown(mbedtls_platform_context *ctx); - -#ifdef __cplusplus -} -#endif - -#endif /* platform.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/platform_time.h b/android/mbedtls/arm64-v8a/include/mbedtls/platform_time.h deleted file mode 100644 index 97f1963ab..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/platform_time.h +++ /dev/null @@ -1,79 +0,0 @@ -/** - * \file platform_time.h - * - * \brief Mbed TLS Platform time abstraction - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PLATFORM_TIME_H -#define MBEDTLS_PLATFORM_TIME_H - -#include "mbedtls/build_info.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * The time_t datatype - */ -#if defined(MBEDTLS_PLATFORM_TIME_TYPE_MACRO) -typedef MBEDTLS_PLATFORM_TIME_TYPE_MACRO mbedtls_time_t; -#else -/* For time_t */ -#include -typedef time_t mbedtls_time_t; -#endif /* MBEDTLS_PLATFORM_TIME_TYPE_MACRO */ - -#if defined(MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO) -typedef MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO mbedtls_ms_time_t; -#else -#include -#include -typedef int64_t mbedtls_ms_time_t; -#endif /* MBEDTLS_PLATFORM_MS_TIME_TYPE_MACRO */ - -/** - * \brief Get time in milliseconds. - * - * \return Monotonically-increasing current time in milliseconds. - * - * \note Define MBEDTLS_PLATFORM_MS_TIME_ALT to be able to provide an - * alternative implementation - * - * \warning This function returns a monotonically-increasing time value from a - * start time that will differ from platform to platform, and possibly - * from run to run of the process. - * - */ -mbedtls_ms_time_t mbedtls_ms_time(void); - -/* - * The function pointers for time - */ -#if defined(MBEDTLS_PLATFORM_TIME_ALT) -extern mbedtls_time_t (*mbedtls_time)(mbedtls_time_t *time); - -/** - * \brief Set your own time function pointer - * - * \param time_func the time function implementation - * - * \return 0 - */ -int mbedtls_platform_set_time(mbedtls_time_t (*time_func)(mbedtls_time_t *time)); -#else -#if defined(MBEDTLS_PLATFORM_TIME_MACRO) -#define mbedtls_time MBEDTLS_PLATFORM_TIME_MACRO -#else -#define mbedtls_time time -#endif /* MBEDTLS_PLATFORM_TIME_MACRO */ -#endif /* MBEDTLS_PLATFORM_TIME_ALT */ - -#ifdef __cplusplus -} -#endif - -#endif /* platform_time.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/platform_util.h b/android/mbedtls/arm64-v8a/include/mbedtls/platform_util.h deleted file mode 100644 index adad6bc3f..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/platform_util.h +++ /dev/null @@ -1,247 +0,0 @@ -/** - * \file platform_util.h - * - * \brief Common and shared functions used by multiple modules in the Mbed TLS - * library. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_PLATFORM_UTIL_H -#define MBEDTLS_PLATFORM_UTIL_H - -#include "mbedtls/build_info.h" - -#include -#if defined(MBEDTLS_HAVE_TIME_DATE) -#include "mbedtls/platform_time.h" -#include -#endif /* MBEDTLS_HAVE_TIME_DATE */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Internal helper macros for deprecating API constants. */ -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -#if defined(MBEDTLS_DEPRECATED_WARNING) -#define MBEDTLS_DEPRECATED __attribute__((deprecated)) -MBEDTLS_DEPRECATED typedef char const *mbedtls_deprecated_string_constant_t; -#define MBEDTLS_DEPRECATED_STRING_CONSTANT(VAL) \ - ((mbedtls_deprecated_string_constant_t) (VAL)) -MBEDTLS_DEPRECATED typedef int mbedtls_deprecated_numeric_constant_t; -#define MBEDTLS_DEPRECATED_NUMERIC_CONSTANT(VAL) \ - ((mbedtls_deprecated_numeric_constant_t) (VAL)) -#else /* MBEDTLS_DEPRECATED_WARNING */ -#define MBEDTLS_DEPRECATED -#define MBEDTLS_DEPRECATED_STRING_CONSTANT(VAL) VAL -#define MBEDTLS_DEPRECATED_NUMERIC_CONSTANT(VAL) VAL -#endif /* MBEDTLS_DEPRECATED_WARNING */ -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/* Implementation of the check-return facility. - * See the user documentation in mbedtls_config.h. - * - * Do not use this macro directly to annotate function: instead, - * use one of MBEDTLS_CHECK_RETURN_CRITICAL or MBEDTLS_CHECK_RETURN_TYPICAL - * depending on how important it is to check the return value. - */ -#if !defined(MBEDTLS_CHECK_RETURN) -#if defined(__GNUC__) -#define MBEDTLS_CHECK_RETURN __attribute__((__warn_unused_result__)) -#elif defined(_MSC_VER) && _MSC_VER >= 1700 -#include -#define MBEDTLS_CHECK_RETURN _Check_return_ -#else -#define MBEDTLS_CHECK_RETURN -#endif -#endif - -/** Critical-failure function - * - * This macro appearing at the beginning of the declaration of a function - * indicates that its return value should be checked in all applications. - * Omitting the check is very likely to indicate a bug in the application - * and will result in a compile-time warning if #MBEDTLS_CHECK_RETURN - * is implemented for the compiler in use. - * - * \note The use of this macro is a work in progress. - * This macro may be added to more functions in the future. - * Such an extension is not considered an API break, provided that - * there are near-unavoidable circumstances under which the function - * can fail. For example, signature/MAC/AEAD verification functions, - * and functions that require a random generator, are considered - * return-check-critical. - */ -#define MBEDTLS_CHECK_RETURN_CRITICAL MBEDTLS_CHECK_RETURN - -/** Ordinary-failure function - * - * This macro appearing at the beginning of the declaration of a function - * indicates that its return value should be generally be checked in portable - * applications. Omitting the check will result in a compile-time warning if - * #MBEDTLS_CHECK_RETURN is implemented for the compiler in use and - * #MBEDTLS_CHECK_RETURN_WARNING is enabled in the compile-time configuration. - * - * You can use #MBEDTLS_IGNORE_RETURN to explicitly ignore the return value - * of a function that is annotated with #MBEDTLS_CHECK_RETURN. - * - * \note The use of this macro is a work in progress. - * This macro will be added to more functions in the future. - * Eventually this should appear before most functions returning - * an error code (as \c int in the \c mbedtls_xxx API or - * as ::psa_status_t in the \c psa_xxx API). - */ -#if defined(MBEDTLS_CHECK_RETURN_WARNING) -#define MBEDTLS_CHECK_RETURN_TYPICAL MBEDTLS_CHECK_RETURN -#else -#define MBEDTLS_CHECK_RETURN_TYPICAL -#endif - -/** Benign-failure function - * - * This macro appearing at the beginning of the declaration of a function - * indicates that it is rarely useful to check its return value. - * - * This macro has an empty expansion. It exists for documentation purposes: - * a #MBEDTLS_CHECK_RETURN_OPTIONAL annotation indicates that the function - * has been analyzed for return-check usefulness, whereas the lack of - * an annotation indicates that the function has not been analyzed and its - * return-check usefulness is unknown. - */ -#define MBEDTLS_CHECK_RETURN_OPTIONAL - -/** \def MBEDTLS_IGNORE_RETURN - * - * Call this macro with one argument, a function call, to suppress a warning - * from #MBEDTLS_CHECK_RETURN due to that function call. - */ -#if !defined(MBEDTLS_IGNORE_RETURN) -/* GCC doesn't silence the warning with just (void)(result). - * (void)!(result) is known to work up at least up to GCC 10, as well - * as with Clang and MSVC. - * - * https://gcc.gnu.org/onlinedocs/gcc-3.4.6/gcc/Non_002dbugs.html - * https://stackoverflow.com/questions/40576003/ignoring-warning-wunused-result - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66425#c34 - */ -#define MBEDTLS_IGNORE_RETURN(result) ((void) !(result)) -#endif - -/* If the following macro is defined, the library is being built by the test - * framework, and the framework is going to provide a replacement - * mbedtls_platform_zeroize() using a preprocessor macro, so the function - * declaration should be omitted. */ -#if !defined(MBEDTLS_TEST_DEFINES_ZEROIZE) //no-check-names -/** - * \brief Securely zeroize a buffer - * - * The function is meant to wipe the data contained in a buffer so - * that it can no longer be recovered even if the program memory - * is later compromised. Call this function on sensitive data - * stored on the stack before returning from a function, and on - * sensitive data stored on the heap before freeing the heap - * object. - * - * It is extremely difficult to guarantee that calls to - * mbedtls_platform_zeroize() are not removed by aggressive - * compiler optimizations in a portable way. For this reason, Mbed - * TLS provides the configuration option - * MBEDTLS_PLATFORM_ZEROIZE_ALT, which allows users to configure - * mbedtls_platform_zeroize() to use a suitable implementation for - * their platform and needs - * - * \param buf Buffer to be zeroized - * \param len Length of the buffer in bytes - * - */ -void mbedtls_platform_zeroize(void *buf, size_t len); -#endif - -/** \brief The type of custom random generator (RNG) callbacks. - * - * Many Mbed TLS functions take two parameters - * `mbedtls_f_rng_t *f_rng, void *p_rng`. The - * library will call \c f_rng to generate - * random values. - * - * \note This is typically one of the following: - * - mbedtls_ctr_drbg_random() with \c p_rng - * pointing to a #mbedtls_ctr_drbg_context; - * - mbedtls_hmac_drbg_random() with \c p_rng - * pointing to a #mbedtls_hmac_drbg_context; - * - mbedtls_psa_get_random() with - * `prng = MBEDTLS_PSA_RANDOM_STATE`. - * - * \note Generally, given a call - * `mbedtls_foo(f_rng, p_rng, ....)`, the RNG callback - * and the context only need to remain valid until - * the call to `mbedtls_foo` returns. However, there - * are a few exceptions where the callback is stored - * in for future use. Check the documentation of - * the calling function. - * - * \warning In a multithreaded environment, calling the - * function should be thread-safe. The standard - * functions provided by the library are thread-safe - * when #MBEDTLS_THREADING_C is enabled. - * - * \warning This function must either provide as many - * bytes as requested of **cryptographic quality** - * random data, or return a negative error code. - * - * \param p_rng The \c p_rng argument that was passed along \c f_rng. - * The library always passes \c p_rng unchanged. - * This is typically a pointer to the random generator - * state, or \c NULL if the custom random generator - * doesn't need a context-specific state. - * \param[out] output On success, this must be filled with \p output_size - * bytes of cryptographic-quality random data. - * \param output_size The number of bytes to output. - * - * \return \c 0 on success, or a negative error code on failure. - * Library functions will generally propagate this - * error code, so \c MBEDTLS_ERR_xxx values are - * recommended. #MBEDTLS_ERR_ENTROPY_SOURCE_FAILED is - * typically sensible for RNG failures. - */ -typedef int mbedtls_f_rng_t(void *p_rng, - unsigned char *output, size_t output_size); - -#if defined(MBEDTLS_HAVE_TIME_DATE) -/** - * \brief Platform-specific implementation of gmtime_r() - * - * The function is a thread-safe abstraction that behaves - * similarly to the gmtime_r() function from Unix/POSIX. - * - * Mbed TLS will try to identify the underlying platform and - * make use of an appropriate underlying implementation (e.g. - * gmtime_r() for POSIX and gmtime_s() for Windows). If this is - * not possible, then gmtime() will be used. In this case, calls - * from the library to gmtime() will be guarded by the mutex - * mbedtls_threading_gmtime_mutex if MBEDTLS_THREADING_C is - * enabled. It is recommended that calls from outside the library - * are also guarded by this mutex. - * - * If MBEDTLS_PLATFORM_GMTIME_R_ALT is defined, then Mbed TLS will - * unconditionally use the alternative implementation for - * mbedtls_platform_gmtime_r() supplied by the user at compile time. - * - * \param tt Pointer to an object containing time (in seconds) since the - * epoch to be converted - * \param tm_buf Pointer to an object where the results will be stored - * - * \return Pointer to an object of type struct tm on success, otherwise - * NULL - */ -struct tm *mbedtls_platform_gmtime_r(const mbedtls_time_t *tt, - struct tm *tm_buf); -#endif /* MBEDTLS_HAVE_TIME_DATE */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_PLATFORM_UTIL_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/poly1305.h b/android/mbedtls/arm64-v8a/include/mbedtls/poly1305.h deleted file mode 100644 index 61bcaa6b6..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/poly1305.h +++ /dev/null @@ -1,168 +0,0 @@ -/** - * \file poly1305.h - * - * \brief This file contains Poly1305 definitions and functions. - * - * Poly1305 is a one-time message authenticator that can be used to - * authenticate messages. Poly1305-AES was created by Daniel - * Bernstein https://cr.yp.to/mac/poly1305-20050329.pdf The generic - * Poly1305 algorithm (not tied to AES) was also standardized in RFC - * 7539. - * - * \author Daniel King - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_POLY1305_H -#define MBEDTLS_POLY1305_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -/** Invalid input parameter(s). */ -#define MBEDTLS_ERR_POLY1305_BAD_INPUT_DATA -0x0057 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_POLY1305_ALT) - -typedef struct mbedtls_poly1305_context { - uint32_t MBEDTLS_PRIVATE(r)[4]; /** The value for 'r' (low 128 bits of the key). */ - uint32_t MBEDTLS_PRIVATE(s)[4]; /** The value for 's' (high 128 bits of the key). */ - uint32_t MBEDTLS_PRIVATE(acc)[5]; /** The accumulator number. */ - uint8_t MBEDTLS_PRIVATE(queue)[16]; /** The current partial block of data. */ - size_t MBEDTLS_PRIVATE(queue_len); /** The number of bytes stored in 'queue'. */ -} -mbedtls_poly1305_context; - -#else /* MBEDTLS_POLY1305_ALT */ -#include "poly1305_alt.h" -#endif /* MBEDTLS_POLY1305_ALT */ - -/** - * \brief This function initializes the specified Poly1305 context. - * - * It must be the first API called before using - * the context. - * - * It is usually followed by a call to - * \c mbedtls_poly1305_starts(), then one or more calls to - * \c mbedtls_poly1305_update(), then one call to - * \c mbedtls_poly1305_finish(), then finally - * \c mbedtls_poly1305_free(). - * - * \param ctx The Poly1305 context to initialize. This must - * not be \c NULL. - */ -void mbedtls_poly1305_init(mbedtls_poly1305_context *ctx); - -/** - * \brief This function releases and clears the specified - * Poly1305 context. - * - * \param ctx The Poly1305 context to clear. This may be \c NULL, in which - * case this function is a no-op. If it is not \c NULL, it must - * point to an initialized Poly1305 context. - */ -void mbedtls_poly1305_free(mbedtls_poly1305_context *ctx); - -/** - * \brief This function sets the one-time authentication key. - * - * \warning The key must be unique and unpredictable for each - * invocation of Poly1305. - * - * \param ctx The Poly1305 context to which the key should be bound. - * This must be initialized. - * \param key The buffer containing the \c 32 Byte (\c 256 Bit) key. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_poly1305_starts(mbedtls_poly1305_context *ctx, - const unsigned char key[32]); - -/** - * \brief This functions feeds an input buffer into an ongoing - * Poly1305 computation. - * - * It is called between \c mbedtls_cipher_poly1305_starts() and - * \c mbedtls_cipher_poly1305_finish(). - * It can be called repeatedly to process a stream of data. - * - * \param ctx The Poly1305 context to use for the Poly1305 operation. - * This must be initialized and bound to a key. - * \param ilen The length of the input data in Bytes. - * Any value is accepted. - * \param input The buffer holding the input data. - * This pointer can be \c NULL if `ilen == 0`. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_poly1305_update(mbedtls_poly1305_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief This function generates the Poly1305 Message - * Authentication Code (MAC). - * - * \param ctx The Poly1305 context to use for the Poly1305 operation. - * This must be initialized and bound to a key. - * \param mac The buffer to where the MAC is written. This must - * be a writable buffer of length \c 16 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_poly1305_finish(mbedtls_poly1305_context *ctx, - unsigned char mac[16]); - -/** - * \brief This function calculates the Poly1305 MAC of the input - * buffer with the provided key. - * - * \warning The key must be unique and unpredictable for each - * invocation of Poly1305. - * - * \param key The buffer containing the \c 32 Byte (\c 256 Bit) key. - * \param ilen The length of the input data in Bytes. - * Any value is accepted. - * \param input The buffer holding the input data. - * This pointer can be \c NULL if `ilen == 0`. - * \param mac The buffer to where the MAC is written. This must be - * a writable buffer of length \c 16 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_poly1305_mac(const unsigned char key[32], - const unsigned char *input, - size_t ilen, - unsigned char mac[16]); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief The Poly1305 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_poly1305_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_POLY1305_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/private_access.h b/android/mbedtls/arm64-v8a/include/mbedtls/private_access.h deleted file mode 100644 index 580f3eb44..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/private_access.h +++ /dev/null @@ -1,20 +0,0 @@ -/** - * \file private_access.h - * - * \brief Macro wrapper for struct's members. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_PRIVATE_ACCESS_H -#define MBEDTLS_PRIVATE_ACCESS_H - -#ifndef MBEDTLS_ALLOW_PRIVATE_ACCESS -#define MBEDTLS_PRIVATE(member) private_##member -#else -#define MBEDTLS_PRIVATE(member) member -#endif - -#endif /* MBEDTLS_PRIVATE_ACCESS_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/psa_util.h b/android/mbedtls/arm64-v8a/include/mbedtls/psa_util.h deleted file mode 100644 index b898f1f8d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/psa_util.h +++ /dev/null @@ -1,207 +0,0 @@ -/** - * \file psa_util.h - * - * \brief Utility functions for the use of the PSA Crypto library. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_PSA_UTIL_H -#define MBEDTLS_PSA_UTIL_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "psa/crypto.h" - -/* ASN1 defines used in the ECDSA conversion functions. - * Note: intentionally not adding MBEDTLS_ASN1_[PARSE|WRITE]_C guards here - * otherwise error codes would be unknown in test_suite_psa_crypto_util.data.*/ -#include - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) - -/** The random generator function for the PSA subsystem. - * - * This function is suitable as the `f_rng` random generator function - * parameter of many `mbedtls_xxx` functions. - * - * The implementation of this function depends on the configuration of the - * library. - * - * \note This function may only be used if the PSA crypto subsystem is active. - * This means that you must call psa_crypto_init() before any call to - * this function, and you must not call this function after calling - * mbedtls_psa_crypto_free(). - * - * \param p_rng This parameter is only kept for backward compatibility - * reasons with legacy `f_rng` functions and it's ignored. - * Set to #MBEDTLS_PSA_RANDOM_STATE or NULL. - * \param output The buffer to fill. It must have room for - * \c output_size bytes. - * \param output_size The number of bytes to write to \p output. - * This function may fail if \p output_size is too - * large. It is guaranteed to accept any output size - * requested by Mbed TLS library functions. The - * maximum request size depends on the library - * configuration. - * - * \return \c 0 on success. - * \return An `MBEDTLS_ERR_ENTROPY_xxx`, - * `MBEDTLS_ERR_PLATFORM_xxx, - * `MBEDTLS_ERR_CTR_DRBG_xxx` or - * `MBEDTLS_ERR_HMAC_DRBG_xxx` on error. - */ -int mbedtls_psa_get_random(void *p_rng, - unsigned char *output, - size_t output_size); - -/** The random generator state for the PSA subsystem. - * - * This macro always expands to NULL because the `p_rng` parameter is unused - * in mbedtls_psa_get_random(), but it's kept for interface's backward - * compatibility. - */ -#define MBEDTLS_PSA_RANDOM_STATE NULL - -/** \defgroup psa_tls_helpers TLS helper functions - * @{ - */ -#if defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) -#include - -/** Convert an ECC curve identifier from the Mbed TLS encoding to PSA. - * - * \param grpid An Mbed TLS elliptic curve identifier - * (`MBEDTLS_ECP_DP_xxx`). - * \param[out] bits On success the bit size of the curve; 0 on failure. - * - * \return If the curve is supported in the PSA API, this function - * returns the proper PSA curve identifier - * (`PSA_ECC_FAMILY_xxx`). This holds even if the curve is - * not supported by the ECP module. - * \return \c 0 if the curve is not supported in the PSA API. - */ -psa_ecc_family_t mbedtls_ecc_group_to_psa(mbedtls_ecp_group_id grpid, - size_t *bits); - -/** Convert an ECC curve identifier from the PSA encoding to Mbed TLS. - * - * \param family A PSA elliptic curve family identifier - * (`PSA_ECC_FAMILY_xxx`). - * \param bits The bit-length of a private key on \p curve. - * - * \return If the curve is supported in the PSA API, this function - * returns the corresponding Mbed TLS elliptic curve - * identifier (`MBEDTLS_ECP_DP_xxx`). - * \return #MBEDTLS_ECP_DP_NONE if the combination of \c curve - * and \p bits is not supported. - */ -mbedtls_ecp_group_id mbedtls_ecc_group_from_psa(psa_ecc_family_t family, - size_t bits); -#endif /* PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY */ - -/** - * \brief This function returns the PSA algorithm identifier - * associated with the given digest type. - * - * \param md_type The type of digest to search for. Must not be NONE. - * - * \warning If \p md_type is \c MBEDTLS_MD_NONE, this function will - * not return \c PSA_ALG_NONE, but an invalid algorithm. - * - * \warning This function does not check if the algorithm is - * supported, it always returns the corresponding identifier. - * - * \return The PSA algorithm identifier associated with \p md_type, - * regardless of whether it is supported or not. - */ -static inline psa_algorithm_t mbedtls_md_psa_alg_from_type(mbedtls_md_type_t md_type) -{ - return PSA_ALG_CATEGORY_HASH | (psa_algorithm_t) md_type; -} - -/** - * \brief This function returns the given digest type - * associated with the PSA algorithm identifier. - * - * \param psa_alg The PSA algorithm identifier to search for. - * - * \warning This function does not check if the algorithm is - * supported, it always returns the corresponding identifier. - * - * \return The MD type associated with \p psa_alg, - * regardless of whether it is supported or not. - */ -static inline mbedtls_md_type_t mbedtls_md_type_from_psa_alg(psa_algorithm_t psa_alg) -{ - return (mbedtls_md_type_t) (psa_alg & PSA_ALG_HASH_MASK); -} -#endif /* MBEDTLS_PSA_CRYPTO_CLIENT */ - -#if defined(MBEDTLS_PSA_UTIL_HAVE_ECDSA) - -/** Convert an ECDSA signature from raw format to DER ASN.1 format. - * - * \param bits Size of each coordinate in bits. - * \param raw Buffer that contains the signature in raw format. - * \param raw_len Length of \p raw in bytes. This must be - * PSA_BITS_TO_BYTES(bits) bytes. - * \param[out] der Buffer that will be filled with the converted DER - * output. It can overlap with raw buffer. - * \param der_size Size of \p der in bytes. It is enough if \p der_size - * is at least the size of the actual output. (The size - * of the output can vary depending on the presence of - * leading zeros in the data.) You can use - * #MBEDTLS_ECDSA_MAX_SIG_LEN(\p bits) to determine a - * size that is large enough for all signatures for a - * given value of \p bits. - * \param[out] der_len On success it contains the amount of valid data - * (in bytes) written to \p der. It's undefined - * in case of failure. - * - * \note The behavior is undefined if \p der is null, - * even if \p der_size is 0. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_BUF_TOO_SMALL if \p der_size - * is too small or if \p bits is larger than the - * largest supported curve. - * \return #MBEDTLS_ERR_ASN1_INVALID_DATA if one of the - * numbers in the signature is 0. - */ -int mbedtls_ecdsa_raw_to_der(size_t bits, const unsigned char *raw, size_t raw_len, - unsigned char *der, size_t der_size, size_t *der_len); - -/** Convert an ECDSA signature from DER ASN.1 format to raw format. - * - * \param bits Size of each coordinate in bits. - * \param der Buffer that contains the signature in DER format. - * \param der_len Size of \p der in bytes. - * \param[out] raw Buffer that will be filled with the converted raw - * signature. It can overlap with der buffer. - * \param raw_size Size of \p raw in bytes. Must be at least - * 2 * PSA_BITS_TO_BYTES(bits) bytes. - * \param[out] raw_len On success it is updated with the amount of valid - * data (in bytes) written to \p raw. It's undefined - * in case of failure. - * - * \return 0 if successful. - * \return #MBEDTLS_ERR_ASN1_BUF_TOO_SMALL if \p raw_size - * is too small or if \p bits is larger than the - * largest supported curve. - * \return #MBEDTLS_ERR_ASN1_INVALID_DATA if the data in - * \p der is inconsistent with \p bits. - * \return An \c MBEDTLS_ERR_ASN1_xxx error code if - * \p der is malformed. - */ -int mbedtls_ecdsa_der_to_raw(size_t bits, const unsigned char *der, size_t der_len, - unsigned char *raw, size_t raw_size, size_t *raw_len); - -#endif /* MBEDTLS_PSA_UTIL_HAVE_ECDSA */ - -/**@}*/ - -#endif /* MBEDTLS_PSA_UTIL_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ripemd160.h b/android/mbedtls/arm64-v8a/include/mbedtls/ripemd160.h deleted file mode 100644 index 279f92b51..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ripemd160.h +++ /dev/null @@ -1,136 +0,0 @@ -/** - * \file ripemd160.h - * - * \brief RIPE MD-160 message digest - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_RIPEMD160_H -#define MBEDTLS_RIPEMD160_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_RIPEMD160_ALT) -// Regular implementation -// - -/** - * \brief RIPEMD-160 context structure - */ -typedef struct mbedtls_ripemd160_context { - uint32_t MBEDTLS_PRIVATE(total)[2]; /*!< number of bytes processed */ - uint32_t MBEDTLS_PRIVATE(state)[5]; /*!< intermediate digest state */ - unsigned char MBEDTLS_PRIVATE(buffer)[64]; /*!< data block being processed */ -} -mbedtls_ripemd160_context; - -#else /* MBEDTLS_RIPEMD160_ALT */ -#include "ripemd160_alt.h" -#endif /* MBEDTLS_RIPEMD160_ALT */ - -/** - * \brief Initialize RIPEMD-160 context - * - * \param ctx RIPEMD-160 context to be initialized - */ -void mbedtls_ripemd160_init(mbedtls_ripemd160_context *ctx); - -/** - * \brief Clear RIPEMD-160 context - * - * \param ctx RIPEMD-160 context to be cleared - */ -void mbedtls_ripemd160_free(mbedtls_ripemd160_context *ctx); - -/** - * \brief Clone (the state of) a RIPEMD-160 context - * - * \param dst The destination context - * \param src The context to be cloned - */ -void mbedtls_ripemd160_clone(mbedtls_ripemd160_context *dst, - const mbedtls_ripemd160_context *src); - -/** - * \brief RIPEMD-160 context setup - * - * \param ctx context to be initialized - * - * \return 0 if successful - */ -int mbedtls_ripemd160_starts(mbedtls_ripemd160_context *ctx); - -/** - * \brief RIPEMD-160 process buffer - * - * \param ctx RIPEMD-160 context - * \param input buffer holding the data - * \param ilen length of the input data - * - * \return 0 if successful - */ -int mbedtls_ripemd160_update(mbedtls_ripemd160_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief RIPEMD-160 final digest - * - * \param ctx RIPEMD-160 context - * \param output RIPEMD-160 checksum result - * - * \return 0 if successful - */ -int mbedtls_ripemd160_finish(mbedtls_ripemd160_context *ctx, - unsigned char output[20]); - -/** - * \brief RIPEMD-160 process data block (internal use only) - * - * \param ctx RIPEMD-160 context - * \param data buffer holding one block of data - * - * \return 0 if successful - */ -int mbedtls_internal_ripemd160_process(mbedtls_ripemd160_context *ctx, - const unsigned char data[64]); - -/** - * \brief Output = RIPEMD-160( input buffer ) - * - * \param input buffer holding the data - * \param ilen length of the input data - * \param output RIPEMD-160 checksum result - * - * \return 0 if successful - */ -int mbedtls_ripemd160(const unsigned char *input, - size_t ilen, - unsigned char output[20]); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief Checkup routine - * - * \return 0 if successful, or 1 if the test failed - */ -int mbedtls_ripemd160_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_ripemd160.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/rsa.h b/android/mbedtls/arm64-v8a/include/mbedtls/rsa.h deleted file mode 100644 index 3f0881a43..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/rsa.h +++ /dev/null @@ -1,1164 +0,0 @@ -/** - * \file rsa.h - * - * \brief This file provides an API for the RSA public-key cryptosystem. - * - * The RSA public-key cryptosystem is defined in Public-Key - * Cryptography Standards (PKCS) #1 v1.5: RSA Encryption - * and Public-Key Cryptography Standards (PKCS) #1 v2.1: - * RSA Cryptography Specifications. - * - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_RSA_H -#define MBEDTLS_RSA_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/bignum.h" -#include "mbedtls/md.h" - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -/* - * RSA Error codes - */ -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_RSA_BAD_INPUT_DATA -0x4080 -/** Input data contains invalid padding and is rejected. */ -#define MBEDTLS_ERR_RSA_INVALID_PADDING -0x4100 -/** Something failed during generation of a key. */ -#define MBEDTLS_ERR_RSA_KEY_GEN_FAILED -0x4180 -/** Key failed to pass the validity check of the library. */ -#define MBEDTLS_ERR_RSA_KEY_CHECK_FAILED -0x4200 -/** The public key operation failed. */ -#define MBEDTLS_ERR_RSA_PUBLIC_FAILED -0x4280 -/** The private key operation failed. */ -#define MBEDTLS_ERR_RSA_PRIVATE_FAILED -0x4300 -/** The PKCS#1 verification failed. */ -#define MBEDTLS_ERR_RSA_VERIFY_FAILED -0x4380 -/** The output buffer for decryption is not large enough. */ -#define MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE -0x4400 -/** The random generator failed to generate non-zeros. */ -#define MBEDTLS_ERR_RSA_RNG_FAILED -0x4480 - -/* - * RSA constants - */ - -#define MBEDTLS_RSA_PKCS_V15 0 /**< Use PKCS#1 v1.5 encoding. */ -#define MBEDTLS_RSA_PKCS_V21 1 /**< Use PKCS#1 v2.1 encoding. */ - -#define MBEDTLS_RSA_SIGN 1 /**< Identifier for RSA signature operations. */ -#define MBEDTLS_RSA_CRYPT 2 /**< Identifier for RSA encryption and decryption operations. */ - -#define MBEDTLS_RSA_SALT_LEN_ANY -1 - -/* - * The above constants may be used even if the RSA module is compile out, - * eg for alternative (PKCS#11) RSA implementations in the PK layers. - */ - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_RSA_ALT) -// Regular implementation -// - -#if !defined(MBEDTLS_RSA_GEN_KEY_MIN_BITS) -#define MBEDTLS_RSA_GEN_KEY_MIN_BITS 1024 -#elif MBEDTLS_RSA_GEN_KEY_MIN_BITS < 128 -#error "MBEDTLS_RSA_GEN_KEY_MIN_BITS must be at least 128 bits" -#endif - -/** - * \brief The RSA context structure. - */ -typedef struct mbedtls_rsa_context { - int MBEDTLS_PRIVATE(ver); /*!< Reserved for internal purposes. - * Do not set this field in application - * code. Its meaning might change without - * notice. */ - size_t MBEDTLS_PRIVATE(len); /*!< The size of \p N in Bytes. */ - - mbedtls_mpi MBEDTLS_PRIVATE(N); /*!< The public modulus. */ - mbedtls_mpi MBEDTLS_PRIVATE(E); /*!< The public exponent. */ - - mbedtls_mpi MBEDTLS_PRIVATE(D); /*!< The private exponent. */ - mbedtls_mpi MBEDTLS_PRIVATE(P); /*!< The first prime factor. */ - mbedtls_mpi MBEDTLS_PRIVATE(Q); /*!< The second prime factor. */ - - mbedtls_mpi MBEDTLS_PRIVATE(DP); /*!< D % (P - 1). */ - mbedtls_mpi MBEDTLS_PRIVATE(DQ); /*!< D % (Q - 1). */ - mbedtls_mpi MBEDTLS_PRIVATE(QP); /*!< 1 / (Q % P). */ - - mbedtls_mpi MBEDTLS_PRIVATE(RN); /*!< cached R^2 mod N. */ - - mbedtls_mpi MBEDTLS_PRIVATE(RP); /*!< cached R^2 mod P. */ - mbedtls_mpi MBEDTLS_PRIVATE(RQ); /*!< cached R^2 mod Q. */ - - mbedtls_mpi MBEDTLS_PRIVATE(Vi); /*!< The cached blinding value. */ - mbedtls_mpi MBEDTLS_PRIVATE(Vf); /*!< The cached un-blinding value. */ - - int MBEDTLS_PRIVATE(padding); /*!< Selects padding mode: - #MBEDTLS_RSA_PKCS_V15 for 1.5 padding and - #MBEDTLS_RSA_PKCS_V21 for OAEP or PSS. */ - int MBEDTLS_PRIVATE(hash_id); /*!< Hash identifier of mbedtls_md_type_t type, - as specified in md.h for use in the MGF - mask generating function used in the - EME-OAEP and EMSA-PSS encodings. */ -#if defined(MBEDTLS_THREADING_C) - /* Invariant: the mutex is initialized iff ver != 0. */ - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); /*!< Thread-safety mutex. */ -#endif -} -mbedtls_rsa_context; - -#else /* MBEDTLS_RSA_ALT */ -#include "rsa_alt.h" -#endif /* MBEDTLS_RSA_ALT */ - -/** - * \brief This function initializes an RSA context. - * - * \note This function initializes the padding and the hash - * identifier to respectively #MBEDTLS_RSA_PKCS_V15 and - * #MBEDTLS_MD_NONE. See mbedtls_rsa_set_padding() for more - * information about those parameters. - * - * \param ctx The RSA context to initialize. This must not be \c NULL. - */ -void mbedtls_rsa_init(mbedtls_rsa_context *ctx); - -/** - * \brief This function sets padding for an already initialized RSA - * context. - * - * \note Set padding to #MBEDTLS_RSA_PKCS_V21 for the RSAES-OAEP - * encryption scheme and the RSASSA-PSS signature scheme. - * - * \note The \p hash_id parameter is ignored when using - * #MBEDTLS_RSA_PKCS_V15 padding. - * - * \note The choice of padding mode is strictly enforced for private - * key operations, since there might be security concerns in - * mixing padding modes. For public key operations it is - * a default value, which can be overridden by calling specific - * \c mbedtls_rsa_rsaes_xxx or \c mbedtls_rsa_rsassa_xxx - * functions. - * - * \note The hash selected in \p hash_id is always used for OEAP - * encryption. For PSS signatures, it is always used for - * making signatures, but can be overridden for verifying them. - * If set to #MBEDTLS_MD_NONE, it is always overridden. - * - * \param ctx The initialized RSA context to be configured. - * \param padding The padding mode to use. This must be either - * #MBEDTLS_RSA_PKCS_V15 or #MBEDTLS_RSA_PKCS_V21. - * \param hash_id The hash identifier for PSS or OAEP, if \p padding is - * #MBEDTLS_RSA_PKCS_V21. #MBEDTLS_MD_NONE is accepted by this - * function but may be not suitable for some operations. - * Ignored if \p padding is #MBEDTLS_RSA_PKCS_V15. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_RSA_INVALID_PADDING failure: - * \p padding or \p hash_id is invalid. - */ -int mbedtls_rsa_set_padding(mbedtls_rsa_context *ctx, int padding, - mbedtls_md_type_t hash_id); - -/** - * \brief This function retrieves padding mode of initialized - * RSA context. - * - * \param ctx The initialized RSA context. - * - * \return RSA padding mode. - * - */ -int mbedtls_rsa_get_padding_mode(const mbedtls_rsa_context *ctx); - -/** - * \brief This function retrieves hash identifier of mbedtls_md_type_t - * type. - * - * \param ctx The initialized RSA context. - * - * \return Hash identifier of mbedtls_md_type_t type. - * - */ -int mbedtls_rsa_get_md_alg(const mbedtls_rsa_context *ctx); - -/** - * \brief This function imports a set of core parameters into an - * RSA context. - * - * \note This function can be called multiple times for successive - * imports, if the parameters are not simultaneously present. - * - * Any sequence of calls to this function should be followed - * by a call to mbedtls_rsa_complete(), which checks and - * completes the provided information to a ready-for-use - * public or private RSA key. - * - * \note See mbedtls_rsa_complete() for more information on which - * parameters are necessary to set up a private or public - * RSA key. - * - * \note The imported parameters are copied and need not be preserved - * for the lifetime of the RSA context being set up. - * - * \param ctx The initialized RSA context to store the parameters in. - * \param N The RSA modulus. This may be \c NULL. - * \param P The first prime factor of \p N. This may be \c NULL. - * \param Q The second prime factor of \p N. This may be \c NULL. - * \param D The private exponent. This may be \c NULL. - * \param E The public exponent. This may be \c NULL. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_rsa_import(mbedtls_rsa_context *ctx, - const mbedtls_mpi *N, - const mbedtls_mpi *P, const mbedtls_mpi *Q, - const mbedtls_mpi *D, const mbedtls_mpi *E); - -/** - * \brief This function imports core RSA parameters, in raw big-endian - * binary format, into an RSA context. - * - * \note This function can be called multiple times for successive - * imports, if the parameters are not simultaneously present. - * - * Any sequence of calls to this function should be followed - * by a call to mbedtls_rsa_complete(), which checks and - * completes the provided information to a ready-for-use - * public or private RSA key. - * - * \note See mbedtls_rsa_complete() for more information on which - * parameters are necessary to set up a private or public - * RSA key. - * - * \note The imported parameters are copied and need not be preserved - * for the lifetime of the RSA context being set up. - * - * \param ctx The initialized RSA context to store the parameters in. - * \param N The RSA modulus. This may be \c NULL. - * \param N_len The Byte length of \p N; it is ignored if \p N == NULL. - * \param P The first prime factor of \p N. This may be \c NULL. - * \param P_len The Byte length of \p P; it is ignored if \p P == NULL. - * \param Q The second prime factor of \p N. This may be \c NULL. - * \param Q_len The Byte length of \p Q; it is ignored if \p Q == NULL. - * \param D The private exponent. This may be \c NULL. - * \param D_len The Byte length of \p D; it is ignored if \p D == NULL. - * \param E The public exponent. This may be \c NULL. - * \param E_len The Byte length of \p E; it is ignored if \p E == NULL. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - */ -int mbedtls_rsa_import_raw(mbedtls_rsa_context *ctx, - unsigned char const *N, size_t N_len, - unsigned char const *P, size_t P_len, - unsigned char const *Q, size_t Q_len, - unsigned char const *D, size_t D_len, - unsigned char const *E, size_t E_len); - -/** - * \brief This function completes an RSA context from - * a set of imported core parameters. - * - * To setup an RSA public key, precisely \c N and \c E - * must have been imported. - * - * To setup an RSA private key, sufficient information must - * be present for the other parameters to be derivable. - * - * The default implementation supports the following: - *
  • Derive \c P, \c Q from \c N, \c D, \c E.
    • - *
  • Derive \c N, \c D from \c P, \c Q, \c E.
- * Alternative implementations need not support these. - * - * If this function runs successfully, it guarantees that - * the RSA context can be used for RSA operations without - * the risk of failure or crash. - * - * \warning This function need not perform consistency checks - * for the imported parameters. In particular, parameters that - * are not needed by the implementation might be silently - * discarded and left unchecked. To check the consistency - * of the key material, see mbedtls_rsa_check_privkey(). - * - * \param ctx The initialized RSA context holding imported parameters. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_RSA_BAD_INPUT_DATA if the attempted derivations - * failed. - * - */ -int mbedtls_rsa_complete(mbedtls_rsa_context *ctx); - -/** - * \brief This function exports the core parameters of an RSA key. - * - * If this function runs successfully, the non-NULL buffers - * pointed to by \p N, \p P, \p Q, \p D, and \p E are fully - * written, with additional unused space filled leading by - * zero Bytes. - * - * Possible reasons for returning - * #MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED:
    - *
  • An alternative RSA implementation is in use, which - * stores the key externally, and either cannot or should - * not export it into RAM.
    • - *
  • A SW or HW implementation might not support a certain - * deduction. For example, \p P, \p Q from \p N, \p D, - * and \p E if the former are not part of the - * implementation.
- * - * If the function fails due to an unsupported operation, - * the RSA context stays intact and remains usable. - * - * \param ctx The initialized RSA context. - * \param N The MPI to hold the RSA modulus. - * This may be \c NULL if this field need not be exported. - * \param P The MPI to hold the first prime factor of \p N. - * This may be \c NULL if this field need not be exported. - * \param Q The MPI to hold the second prime factor of \p N. - * This may be \c NULL if this field need not be exported. - * \param D The MPI to hold the private exponent. - * This may be \c NULL if this field need not be exported. - * \param E The MPI to hold the public exponent. - * This may be \c NULL if this field need not be exported. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED if exporting the - * requested parameters cannot be done due to missing - * functionality or because of security policies. - * \return A non-zero return code on any other failure. - * - */ -int mbedtls_rsa_export(const mbedtls_rsa_context *ctx, - mbedtls_mpi *N, mbedtls_mpi *P, mbedtls_mpi *Q, - mbedtls_mpi *D, mbedtls_mpi *E); - -/** - * \brief This function exports core parameters of an RSA key - * in raw big-endian binary format. - * - * If this function runs successfully, the non-NULL buffers - * pointed to by \p N, \p P, \p Q, \p D, and \p E are fully - * written, with additional unused space filled leading by - * zero Bytes. - * - * Possible reasons for returning - * #MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED:
    - *
  • An alternative RSA implementation is in use, which - * stores the key externally, and either cannot or should - * not export it into RAM.
    • - *
  • A SW or HW implementation might not support a certain - * deduction. For example, \p P, \p Q from \p N, \p D, - * and \p E if the former are not part of the - * implementation.
- * If the function fails due to an unsupported operation, - * the RSA context stays intact and remains usable. - * - * \note The length parameters are ignored if the corresponding - * buffer pointers are NULL. - * - * \param ctx The initialized RSA context. - * \param N The Byte array to store the RSA modulus, - * or \c NULL if this field need not be exported. - * \param N_len The size of the buffer for the modulus. - * \param P The Byte array to hold the first prime factor of \p N, - * or \c NULL if this field need not be exported. - * \param P_len The size of the buffer for the first prime factor. - * \param Q The Byte array to hold the second prime factor of \p N, - * or \c NULL if this field need not be exported. - * \param Q_len The size of the buffer for the second prime factor. - * \param D The Byte array to hold the private exponent, - * or \c NULL if this field need not be exported. - * \param D_len The size of the buffer for the private exponent. - * \param E The Byte array to hold the public exponent, - * or \c NULL if this field need not be exported. - * \param E_len The size of the buffer for the public exponent. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_PLATFORM_FEATURE_UNSUPPORTED if exporting the - * requested parameters cannot be done due to missing - * functionality or because of security policies. - * \return A non-zero return code on any other failure. - */ -int mbedtls_rsa_export_raw(const mbedtls_rsa_context *ctx, - unsigned char *N, size_t N_len, - unsigned char *P, size_t P_len, - unsigned char *Q, size_t Q_len, - unsigned char *D, size_t D_len, - unsigned char *E, size_t E_len); - -/** - * \brief This function exports CRT parameters of a private RSA key. - * - * \note Alternative RSA implementations not using CRT-parameters - * internally can implement this function based on - * mbedtls_rsa_deduce_opt(). - * - * \param ctx The initialized RSA context. - * \param DP The MPI to hold \c D modulo `P-1`, - * or \c NULL if it need not be exported. - * \param DQ The MPI to hold \c D modulo `Q-1`, - * or \c NULL if it need not be exported. - * \param QP The MPI to hold modular inverse of \c Q modulo \c P, - * or \c NULL if it need not be exported. - * - * \return \c 0 on success. - * \return A non-zero error code on failure. - * - */ -int mbedtls_rsa_export_crt(const mbedtls_rsa_context *ctx, - mbedtls_mpi *DP, mbedtls_mpi *DQ, mbedtls_mpi *QP); - -/** - * \brief This function retrieves the length of the RSA modulus in bits. - * - * \param ctx The initialized RSA context. - * - * \return The length of the RSA modulus in bits. - * - */ -size_t mbedtls_rsa_get_bitlen(const mbedtls_rsa_context *ctx); - -/** - * \brief This function retrieves the length of RSA modulus in Bytes. - * - * \param ctx The initialized RSA context. - * - * \return The length of the RSA modulus in Bytes. - * - */ -size_t mbedtls_rsa_get_len(const mbedtls_rsa_context *ctx); - -/** - * \brief This function generates an RSA keypair. - * - * \note mbedtls_rsa_init() must be called before this function, - * to set up the RSA context. - * - * \param ctx The initialized RSA context used to hold the key. - * \param f_rng The RNG function to be used for key generation. - * This is mandatory and must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. - * This may be \c NULL if \p f_rng doesn't need a context. - * \param nbits The size of the public key in bits. - * \param exponent The public exponent to use. For example, \c 65537. - * This must be odd and greater than \c 1. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_gen_key(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - unsigned int nbits, int exponent); - -/** - * \brief This function checks if a context contains at least an RSA - * public key. - * - * If the function runs successfully, it is guaranteed that - * enough information is present to perform an RSA public key - * operation using mbedtls_rsa_public(). - * - * \param ctx The initialized RSA context to check. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - * - */ -int mbedtls_rsa_check_pubkey(const mbedtls_rsa_context *ctx); - -/** - * \brief This function checks if a context contains an RSA private key - * and perform basic consistency checks. - * - * \note The consistency checks performed by this function not only - * ensure that mbedtls_rsa_private() can be called successfully - * on the given context, but that the various parameters are - * mutually consistent with high probability, in the sense that - * mbedtls_rsa_public() and mbedtls_rsa_private() are inverses. - * - * \warning This function should catch accidental misconfigurations - * like swapping of parameters, but it cannot establish full - * trust in neither the quality nor the consistency of the key - * material that was used to setup the given RSA context: - *
  • Consistency: Imported parameters that are irrelevant - * for the implementation might be silently dropped. If dropped, - * the current function does not have access to them, - * and therefore cannot check them. See mbedtls_rsa_complete(). - * If you want to check the consistency of the entire - * content of a PKCS1-encoded RSA private key, for example, you - * should use mbedtls_rsa_validate_params() before setting - * up the RSA context. - * Additionally, if the implementation performs empirical checks, - * these checks substantiate but do not guarantee consistency.
    • - *
  • Quality: This function is not expected to perform - * extended quality assessments like checking that the prime - * factors are safe. Additionally, it is the responsibility of the - * user to ensure the trustworthiness of the source of his RSA - * parameters, which goes beyond what is effectively checkable - * by the library.
- * - * \param ctx The initialized RSA context to check. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_check_privkey(const mbedtls_rsa_context *ctx); - -/** - * \brief This function checks a public-private RSA key pair. - * - * It checks each of the contexts, and makes sure they match. - * - * \param pub The initialized RSA context holding the public key. - * \param prv The initialized RSA context holding the private key. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_check_pub_priv(const mbedtls_rsa_context *pub, - const mbedtls_rsa_context *prv); - -/** - * \brief This function performs an RSA public key operation. - * - * \param ctx The initialized RSA context to use. - * \param input The input buffer. This must be a readable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * \param output The output buffer. This must be a writable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \note This function does not handle message padding. - * - * \note Make sure to set \p input[0] = 0 or ensure that - * input is smaller than \c N. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_public(mbedtls_rsa_context *ctx, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function performs an RSA private key operation. - * - * \note Blinding is used if and only if a PRNG is provided. - * - * \note If blinding is used, both the base of exponentiation - * and the exponent are blinded, providing protection - * against some side-channel attacks. - * - * \warning It is deprecated and a security risk to not provide - * a PRNG here and thereby prevent the use of blinding. - * Future versions of the library may enforce the presence - * of a PRNG. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function, used for blinding. It is mandatory. - * \param p_rng The RNG context to pass to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context. - * \param input The input buffer. This must be a readable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * \param output The output buffer. This must be a writable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - * - */ -int mbedtls_rsa_private(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function adds the message padding, then performs an RSA - * operation. - * - * It is the generic wrapper for performing a PKCS#1 encryption - * operation. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG to use. It is used for padding generation - * and it is mandatory. - * \param p_rng The RNG context to be passed to \p f_rng. May be - * \c NULL if \p f_rng doesn't need a context argument. - * \param ilen The length of the plaintext in Bytes. - * \param input The input data to encrypt. This must be a readable - * buffer of size \p ilen Bytes. It may be \c NULL if - * `ilen == 0`. - * \param output The output buffer. This must be a writable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_pkcs1_encrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - size_t ilen, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function performs a PKCS#1 v1.5 encryption operation - * (RSAES-PKCS1-v1_5-ENCRYPT). - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function to use. It is mandatory and used for - * padding generation. - * \param p_rng The RNG context to be passed to \p f_rng. This may - * be \c NULL if \p f_rng doesn't need a context argument. - * \param ilen The length of the plaintext in Bytes. - * \param input The input data to encrypt. This must be a readable - * buffer of size \p ilen Bytes. It may be \c NULL if - * `ilen == 0`. - * \param output The output buffer. This must be a writable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsaes_pkcs1_v15_encrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - size_t ilen, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function performs a PKCS#1 v2.1 OAEP encryption - * operation (RSAES-OAEP-ENCRYPT). - * - * \note The output buffer must be as large as the size - * of ctx->N. For example, 128 Bytes if RSA-1024 is used. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function to use. This is needed for padding - * generation and is mandatory. - * \param p_rng The RNG context to be passed to \p f_rng. This may - * be \c NULL if \p f_rng doesn't need a context argument. - * \param label The buffer holding the custom label to use. - * This must be a readable buffer of length \p label_len - * Bytes. It may be \c NULL if \p label_len is \c 0. - * \param label_len The length of the label in Bytes. - * \param ilen The length of the plaintext buffer \p input in Bytes. - * \param input The input data to encrypt. This must be a readable - * buffer of size \p ilen Bytes. It may be \c NULL if - * `ilen == 0`. - * \param output The output buffer. This must be a writable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsaes_oaep_encrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - const unsigned char *label, size_t label_len, - size_t ilen, - const unsigned char *input, - unsigned char *output); - -/** - * \brief This function performs an RSA operation, then removes the - * message padding. - * - * It is the generic wrapper for performing a PKCS#1 decryption - * operation. - * - * \warning When \p ctx->padding is set to #MBEDTLS_RSA_PKCS_V15, - * mbedtls_rsa_rsaes_pkcs1_v15_decrypt() is called, which is an - * inherently dangerous function (CWE-242). - * - * \note The output buffer length \c output_max_len should be - * as large as the size \p ctx->len of \p ctx->N (for example, - * 128 Bytes if RSA-1024 is used) to be able to hold an - * arbitrary decrypted message. If it is not large enough to - * hold the decryption of the particular ciphertext provided, - * the function returns \c MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. This is used for blinding and is - * mandatory; see mbedtls_rsa_private() for more. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context. - * \param olen The address at which to store the length of - * the plaintext. This must not be \c NULL. - * \param input The ciphertext buffer. This must be a readable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * \param output The buffer used to hold the plaintext. This must - * be a writable buffer of length \p output_max_len Bytes. - * \param output_max_len The length in Bytes of the output buffer \p output. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_pkcs1_decrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - size_t *olen, - const unsigned char *input, - unsigned char *output, - size_t output_max_len); - -/** - * \brief This function performs a PKCS#1 v1.5 decryption - * operation (RSAES-PKCS1-v1_5-DECRYPT). - * - * \warning This is an inherently dangerous function (CWE-242). Unless - * it is used in a side channel free and safe way (eg. - * implementing the TLS protocol as per 7.4.7.1 of RFC 5246), - * the calling code is vulnerable. - * - * \note The output buffer length \c output_max_len should be - * as large as the size \p ctx->len of \p ctx->N, for example, - * 128 Bytes if RSA-1024 is used, to be able to hold an - * arbitrary decrypted message. If it is not large enough to - * hold the decryption of the particular ciphertext provided, - * the function returns #MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. This is used for blinding and is - * mandatory; see mbedtls_rsa_private() for more. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context. - * \param olen The address at which to store the length of - * the plaintext. This must not be \c NULL. - * \param input The ciphertext buffer. This must be a readable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * \param output The buffer used to hold the plaintext. This must - * be a writable buffer of length \p output_max_len Bytes. - * \param output_max_len The length in Bytes of the output buffer \p output. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - * - */ -int mbedtls_rsa_rsaes_pkcs1_v15_decrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - size_t *olen, - const unsigned char *input, - unsigned char *output, - size_t output_max_len); - -/** - * \brief This function performs a PKCS#1 v2.1 OAEP decryption - * operation (RSAES-OAEP-DECRYPT). - * - * \note The output buffer length \c output_max_len should be - * as large as the size \p ctx->len of \p ctx->N, for - * example, 128 Bytes if RSA-1024 is used, to be able to - * hold an arbitrary decrypted message. If it is not - * large enough to hold the decryption of the particular - * ciphertext provided, the function returns - * #MBEDTLS_ERR_RSA_OUTPUT_TOO_LARGE. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. This is used for blinding and is - * mandatory. - * \param p_rng The RNG context to be passed to \p f_rng. This may be - * \c NULL if \p f_rng doesn't need a context. - * \param label The buffer holding the custom label to use. - * This must be a readable buffer of length \p label_len - * Bytes. It may be \c NULL if \p label_len is \c 0. - * \param label_len The length of the label in Bytes. - * \param olen The address at which to store the length of - * the plaintext. This must not be \c NULL. - * \param input The ciphertext buffer. This must be a readable buffer - * of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * \param output The buffer used to hold the plaintext. This must - * be a writable buffer of length \p output_max_len Bytes. - * \param output_max_len The length in Bytes of the output buffer \p output. - * - * \return \c 0 on success. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsaes_oaep_decrypt(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - const unsigned char *label, size_t label_len, - size_t *olen, - const unsigned char *input, - unsigned char *output, - size_t output_max_len); - -/** - * \brief This function performs a private RSA operation to sign - * a message digest using PKCS#1. - * - * It is the generic wrapper for performing a PKCS#1 - * signature. - * - * \note The \p sig buffer must be as large as the size - * of \p ctx->N. For example, 128 Bytes if RSA-1024 is used. - * - * \note For PKCS#1 v2.1 encoding, see comments on - * mbedtls_rsa_rsassa_pss_sign() for details on - * \p md_alg and \p hash_id. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function to use. This is mandatory and - * must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context argument. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer to hold the signature. This must be a writable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. A buffer length of - * #MBEDTLS_MPI_MAX_SIZE is always safe. - * - * \return \c 0 if the signing operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_pkcs1_sign(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - unsigned char *sig); - -/** - * \brief This function performs a PKCS#1 v1.5 signature - * operation (RSASSA-PKCS1-v1_5-SIGN). - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. This is used for blinding and is - * mandatory; see mbedtls_rsa_private() for more. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context argument. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer to hold the signature. This must be a writable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. A buffer length of - * #MBEDTLS_MPI_MAX_SIZE is always safe. - * - * \return \c 0 if the signing operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pkcs1_v15_sign(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - unsigned char *sig); - -#if defined(MBEDTLS_PKCS1_V21) -/** - * \brief This function performs a PKCS#1 v2.1 PSS signature - * operation (RSASSA-PSS-SIGN). - * - * \note The \c hash_id set in \p ctx by calling - * mbedtls_rsa_set_padding() selects the hash used for the - * encoding operation and for the mask generation function - * (MGF1). For more details on the encoding operation and the - * mask generation function, consult RFC-3447: Public-Key - * Cryptography Standards (PKCS) #1 v2.1: RSA Cryptography - * Specifications. - * - * \note This function enforces that the provided salt length complies - * with FIPS 186-4 §5.5 (e) and RFC 8017 (PKCS#1 v2.2) §9.1.1 - * step 3. The constraint is that the hash length plus the salt - * length plus 2 bytes must be at most the key length. If this - * constraint is not met, this function returns - * #MBEDTLS_ERR_RSA_BAD_INPUT_DATA. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. It is mandatory and must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context argument. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param saltlen The length of the salt that should be used. - * If passed #MBEDTLS_RSA_SALT_LEN_ANY, the function will use - * the largest possible salt length up to the hash length, - * which is the largest permitted by some standards including - * FIPS 186-4 §5.5. - * \param sig The buffer to hold the signature. This must be a writable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. A buffer length of - * #MBEDTLS_MPI_MAX_SIZE is always safe. - * - * \return \c 0 if the signing operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pss_sign_ext(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - int saltlen, - unsigned char *sig); - -/** - * \brief This function performs a PKCS#1 v2.1 PSS signature - * operation (RSASSA-PSS-SIGN). - * - * \note The \c hash_id set in \p ctx by calling - * mbedtls_rsa_set_padding() selects the hash used for the - * encoding operation and for the mask generation function - * (MGF1). For more details on the encoding operation and the - * mask generation function, consult RFC-3447: Public-Key - * Cryptography Standards (PKCS) #1 v2.1: RSA Cryptography - * Specifications. - * - * \note This function always uses the maximum possible salt size, - * up to the length of the payload hash. This choice of salt - * size complies with FIPS 186-4 §5.5 (e) and RFC 8017 (PKCS#1 - * v2.2) §9.1.1 step 3. Furthermore this function enforces a - * minimum salt size which is the hash size minus 2 bytes. If - * this minimum size is too large given the key size (the salt - * size, plus the hash size, plus 2 bytes must be no more than - * the key size in bytes), this function returns - * #MBEDTLS_ERR_RSA_BAD_INPUT_DATA. - * - * \param ctx The initialized RSA context to use. - * \param f_rng The RNG function. It is mandatory and must not be \c NULL. - * \param p_rng The RNG context to be passed to \p f_rng. This may be \c NULL - * if \p f_rng doesn't need a context argument. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer to hold the signature. This must be a writable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. A buffer length of - * #MBEDTLS_MPI_MAX_SIZE is always safe. - * - * \return \c 0 if the signing operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pss_sign(mbedtls_rsa_context *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - unsigned char *sig); -#endif /* MBEDTLS_PKCS1_V21 */ - -/** - * \brief This function performs a public RSA operation and checks - * the message digest. - * - * This is the generic wrapper for performing a PKCS#1 - * verification. - * - * \note For PKCS#1 v2.1 encoding, see comments on - * mbedtls_rsa_rsassa_pss_verify() about \c md_alg and - * \c hash_id. - * - * \param ctx The initialized RSA public key context to use. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer holding the signature. This must be a readable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 if the verify operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_pkcs1_verify(mbedtls_rsa_context *ctx, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - const unsigned char *sig); - -/** - * \brief This function performs a PKCS#1 v1.5 verification - * operation (RSASSA-PKCS1-v1_5-VERIFY). - * - * \param ctx The initialized RSA public key context to use. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer holding the signature. This must be a readable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 if the verify operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pkcs1_v15_verify(mbedtls_rsa_context *ctx, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - const unsigned char *sig); - -/** - * \brief This function performs a PKCS#1 v2.1 PSS verification - * operation (RSASSA-PSS-VERIFY). - * - * \note The \c hash_id set in \p ctx by calling - * mbedtls_rsa_set_padding() selects the hash used for the - * encoding operation and for the mask generation function - * (MGF1). For more details on the encoding operation and the - * mask generation function, consult RFC-3447: Public-Key - * Cryptography Standards (PKCS) #1 v2.1: RSA Cryptography - * Specifications. If the \c hash_id set in \p ctx by - * mbedtls_rsa_set_padding() is #MBEDTLS_MD_NONE, the \p md_alg - * parameter is used. - * - * \param ctx The initialized RSA public key context to use. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param sig The buffer holding the signature. This must be a readable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 if the verify operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pss_verify(mbedtls_rsa_context *ctx, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - const unsigned char *sig); - -/** - * \brief This function performs a PKCS#1 v2.1 PSS verification - * operation (RSASSA-PSS-VERIFY). - * - * \note The \p sig buffer must be as large as the size - * of \p ctx->N. For example, 128 Bytes if RSA-1024 is used. - * - * \note The \c hash_id set in \p ctx by mbedtls_rsa_set_padding() is - * ignored. - * - * \param ctx The initialized RSA public key context to use. - * \param md_alg The message-digest algorithm used to hash the original data. - * Use #MBEDTLS_MD_NONE for signing raw data. - * \param hashlen The length of the message digest or raw data in Bytes. - * If \p md_alg is not #MBEDTLS_MD_NONE, this must match the - * output length of the corresponding hash algorithm. - * \param hash The buffer holding the message digest or raw data. - * This must be a readable buffer of at least \p hashlen Bytes. - * \param mgf1_hash_id The message digest algorithm used for the - * verification operation and the mask generation - * function (MGF1). For more details on the encoding - * operation and the mask generation function, consult - * RFC-3447: Public-Key Cryptography Standards - * (PKCS) #1 v2.1: RSA Cryptography - * Specifications. - * \param expected_salt_len The length of the salt used in padding. Use - * #MBEDTLS_RSA_SALT_LEN_ANY to accept any salt length. - * \param sig The buffer holding the signature. This must be a readable - * buffer of length \c ctx->len Bytes. For example, \c 256 Bytes - * for an 2048-bit RSA modulus. - * - * \return \c 0 if the verify operation was successful. - * \return An \c MBEDTLS_ERR_RSA_XXX error code on failure. - */ -int mbedtls_rsa_rsassa_pss_verify_ext(mbedtls_rsa_context *ctx, - mbedtls_md_type_t md_alg, - unsigned int hashlen, - const unsigned char *hash, - mbedtls_md_type_t mgf1_hash_id, - int expected_salt_len, - const unsigned char *sig); - -/** - * \brief This function copies the components of an RSA context. - * - * \param dst The destination context. This must be initialized. - * \param src The source context. This must be initialized. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_MPI_ALLOC_FAILED on memory allocation failure. - */ -int mbedtls_rsa_copy(mbedtls_rsa_context *dst, const mbedtls_rsa_context *src); - -/** - * \brief This function frees the components of an RSA key. - * - * \param ctx The RSA context to free. May be \c NULL, in which case - * this function is a no-op. If it is not \c NULL, it must - * point to an initialized RSA context. - */ -void mbedtls_rsa_free(mbedtls_rsa_context *ctx); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The RSA checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_rsa_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* rsa.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/sha1.h b/android/mbedtls/arm64-v8a/include/mbedtls/sha1.h deleted file mode 100644 index 592ffd13f..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/sha1.h +++ /dev/null @@ -1,219 +0,0 @@ -/** - * \file sha1.h - * - * \brief This file contains SHA-1 definitions and functions. - * - * The Secure Hash Algorithm 1 (SHA-1) cryptographic hash function is defined in - * FIPS 180-4: Secure Hash Standard (SHS). - * - * \warning SHA-1 is considered a weak message digest and its use constitutes - * a security risk. We recommend considering stronger message - * digests instead. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SHA1_H -#define MBEDTLS_SHA1_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -/** SHA-1 input data was malformed. */ -#define MBEDTLS_ERR_SHA1_BAD_INPUT_DATA -0x0073 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_SHA1_ALT) -// Regular implementation -// - -/** - * \brief The SHA-1 context structure. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - */ -typedef struct mbedtls_sha1_context { - uint32_t MBEDTLS_PRIVATE(total)[2]; /*!< The number of Bytes processed. */ - uint32_t MBEDTLS_PRIVATE(state)[5]; /*!< The intermediate digest state. */ - unsigned char MBEDTLS_PRIVATE(buffer)[64]; /*!< The data block being processed. */ -} -mbedtls_sha1_context; - -#else /* MBEDTLS_SHA1_ALT */ -#include "sha1_alt.h" -#endif /* MBEDTLS_SHA1_ALT */ - -/** - * \brief This function initializes a SHA-1 context. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context to initialize. - * This must not be \c NULL. - * - */ -void mbedtls_sha1_init(mbedtls_sha1_context *ctx); - -/** - * \brief This function clears a SHA-1 context. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context to clear. This may be \c NULL, - * in which case this function does nothing. If it is - * not \c NULL, it must point to an initialized - * SHA-1 context. - * - */ -void mbedtls_sha1_free(mbedtls_sha1_context *ctx); - -/** - * \brief This function clones the state of a SHA-1 context. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param dst The SHA-1 context to clone to. This must be initialized. - * \param src The SHA-1 context to clone from. This must be initialized. - * - */ -void mbedtls_sha1_clone(mbedtls_sha1_context *dst, - const mbedtls_sha1_context *src); - -/** - * \brief This function starts a SHA-1 checksum calculation. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context to initialize. This must be initialized. - * - * \return \c 0 on success. - * \return A negative error code on failure. - * - */ -int mbedtls_sha1_starts(mbedtls_sha1_context *ctx); - -/** - * \brief This function feeds an input buffer into an ongoing SHA-1 - * checksum calculation. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context. This must be initialized - * and have a hash operation started. - * \param input The buffer holding the input data. - * This must be a readable buffer of length \p ilen Bytes. - * \param ilen The length of the input data \p input in Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha1_update(mbedtls_sha1_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief This function finishes the SHA-1 operation, and writes - * the result to the output buffer. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context to use. This must be initialized and - * have a hash operation started. - * \param output The SHA-1 checksum result. This must be a writable - * buffer of length \c 20 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha1_finish(mbedtls_sha1_context *ctx, - unsigned char output[20]); - -/** - * \brief SHA-1 process data block (internal use only). - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param ctx The SHA-1 context to use. This must be initialized. - * \param data The data block being processed. This must be a - * readable buffer of length \c 64 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - * - */ -int mbedtls_internal_sha1_process(mbedtls_sha1_context *ctx, - const unsigned char data[64]); - -/** - * \brief This function calculates the SHA-1 checksum of a buffer. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The SHA-1 result is calculated as - * output = SHA-1(input buffer). - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \param input The buffer holding the input data. - * This must be a readable buffer of length \p ilen Bytes. - * \param ilen The length of the input data \p input in Bytes. - * \param output The SHA-1 checksum result. - * This must be a writable buffer of length \c 20 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - * - */ -int mbedtls_sha1(const unsigned char *input, - size_t ilen, - unsigned char output[20]); - -#if defined(MBEDTLS_SELF_TEST) - -/** - * \brief The SHA-1 checkup routine. - * - * \warning SHA-1 is considered a weak message digest and its use - * constitutes a security risk. We recommend considering - * stronger message digests instead. - * - * \return \c 0 on success. - * \return \c 1 on failure. - * - */ -int mbedtls_sha1_self_test(int verbose); - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_sha1.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/sha256.h b/android/mbedtls/arm64-v8a/include/mbedtls/sha256.h deleted file mode 100644 index ca568e291..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/sha256.h +++ /dev/null @@ -1,200 +0,0 @@ -/** - * \file sha256.h - * - * \brief This file contains SHA-224 and SHA-256 definitions and functions. - * - * The Secure Hash Algorithms 224 and 256 (SHA-224 and SHA-256) cryptographic - * hash functions are defined in FIPS 180-4: Secure Hash Standard (SHS). - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SHA256_H -#define MBEDTLS_SHA256_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -/** SHA-256 input data was malformed. */ -#define MBEDTLS_ERR_SHA256_BAD_INPUT_DATA -0x0074 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_SHA256_ALT) -// Regular implementation -// - -/** - * \brief The SHA-256 context structure. - * - * The structure is used both for SHA-256 and for SHA-224 - * checksum calculations. The choice between these two is - * made in the call to mbedtls_sha256_starts(). - */ -typedef struct mbedtls_sha256_context { - unsigned char MBEDTLS_PRIVATE(buffer)[64]; /*!< The data block being processed. */ - uint32_t MBEDTLS_PRIVATE(total)[2]; /*!< The number of Bytes processed. */ - uint32_t MBEDTLS_PRIVATE(state)[8]; /*!< The intermediate digest state. */ -#if defined(MBEDTLS_SHA224_C) - int MBEDTLS_PRIVATE(is224); /*!< Determines which function to use: - 0: Use SHA-256, or 1: Use SHA-224. */ -#endif -} -mbedtls_sha256_context; - -#else /* MBEDTLS_SHA256_ALT */ -#include "sha256_alt.h" -#endif /* MBEDTLS_SHA256_ALT */ - -/** - * \brief This function initializes a SHA-256 context. - * - * \param ctx The SHA-256 context to initialize. This must not be \c NULL. - */ -void mbedtls_sha256_init(mbedtls_sha256_context *ctx); - -/** - * \brief This function clears a SHA-256 context. - * - * \param ctx The SHA-256 context to clear. This may be \c NULL, in which - * case this function returns immediately. If it is not \c NULL, - * it must point to an initialized SHA-256 context. - */ -void mbedtls_sha256_free(mbedtls_sha256_context *ctx); - -/** - * \brief This function clones the state of a SHA-256 context. - * - * \param dst The destination context. This must be initialized. - * \param src The context to clone. This must be initialized. - */ -void mbedtls_sha256_clone(mbedtls_sha256_context *dst, - const mbedtls_sha256_context *src); - -/** - * \brief This function starts a SHA-224 or SHA-256 checksum - * calculation. - * - * \param ctx The context to use. This must be initialized. - * \param is224 This determines which function to use. This must be - * either \c 0 for SHA-256, or \c 1 for SHA-224. - * - * \note is224 must be defined accordingly to the enabled - * MBEDTLS_SHA224_C/MBEDTLS_SHA256_C symbols otherwise the - * function will return #MBEDTLS_ERR_SHA512_BAD_INPUT_DATA. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha256_starts(mbedtls_sha256_context *ctx, int is224); - -/** - * \brief This function feeds an input buffer into an ongoing - * SHA-256 checksum calculation. - * - * \param ctx The SHA-256 context. This must be initialized - * and have a hash operation started. - * \param input The buffer holding the data. This must be a readable - * buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha256_update(mbedtls_sha256_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief This function finishes the SHA-256 operation, and writes - * the result to the output buffer. - * - * \param ctx The SHA-256 context. This must be initialized - * and have a hash operation started. - * \param output The SHA-224 or SHA-256 checksum result. - * This must be a writable buffer of length \c 32 bytes - * for SHA-256, \c 28 bytes for SHA-224. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha256_finish(mbedtls_sha256_context *ctx, - unsigned char *output); - -/** - * \brief This function processes a single data block within - * the ongoing SHA-256 computation. This function is for - * internal use only. - * - * \param ctx The SHA-256 context. This must be initialized. - * \param data The buffer holding one block of data. This must - * be a readable buffer of length \c 64 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_internal_sha256_process(mbedtls_sha256_context *ctx, - const unsigned char data[64]); - -/** - * \brief This function calculates the SHA-224 or SHA-256 - * checksum of a buffer. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The SHA-256 result is calculated as - * output = SHA-256(input buffer). - * - * \param input The buffer holding the data. This must be a readable - * buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * \param output The SHA-224 or SHA-256 checksum result. - * This must be a writable buffer of length \c 32 bytes - * for SHA-256, \c 28 bytes for SHA-224. - * \param is224 Determines which function to use. This must be - * either \c 0 for SHA-256, or \c 1 for SHA-224. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha256(const unsigned char *input, - size_t ilen, - unsigned char *output, - int is224); - -#if defined(MBEDTLS_SELF_TEST) - -#if defined(MBEDTLS_SHA224_C) -/** - * \brief The SHA-224 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_sha224_self_test(int verbose); -#endif /* MBEDTLS_SHA224_C */ - -#if defined(MBEDTLS_SHA256_C) -/** - * \brief The SHA-256 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_sha256_self_test(int verbose); -#endif /* MBEDTLS_SHA256_C */ - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_sha256.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/sha3.h b/android/mbedtls/arm64-v8a/include/mbedtls/sha3.h deleted file mode 100644 index 3eeee65e6..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/sha3.h +++ /dev/null @@ -1,172 +0,0 @@ -/** - * \file sha3.h - * - * \brief This file contains SHA-3 definitions and functions. - * - * The Secure Hash Algorithms cryptographic - * hash functions are defined in FIPS 202: SHA-3 Standard: - * Permutation-Based Hash and Extendable-Output Functions . - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_SHA3_H -#define MBEDTLS_SHA3_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** SHA-3 input data was malformed. */ -#define MBEDTLS_ERR_SHA3_BAD_INPUT_DATA -0x0076 - -/** - * SHA-3 family id. - * - * It identifies the family (SHA3-256, SHA3-512, etc.) - */ - -typedef enum { - MBEDTLS_SHA3_NONE = 0, /*!< Operation not defined. */ - MBEDTLS_SHA3_224, /*!< SHA3-224 */ - MBEDTLS_SHA3_256, /*!< SHA3-256 */ - MBEDTLS_SHA3_384, /*!< SHA3-384 */ - MBEDTLS_SHA3_512, /*!< SHA3-512 */ -} mbedtls_sha3_id; - -/** - * \brief The SHA-3 context structure. - * - * The structure is used SHA-3 checksum calculations. - */ -typedef struct { - uint64_t MBEDTLS_PRIVATE(state[25]); - uint32_t MBEDTLS_PRIVATE(index); - uint16_t MBEDTLS_PRIVATE(olen); - uint16_t MBEDTLS_PRIVATE(max_block_size); -} -mbedtls_sha3_context; - -/** - * \brief This function initializes a SHA-3 context. - * - * \param ctx The SHA-3 context to initialize. This must not be \c NULL. - */ -void mbedtls_sha3_init(mbedtls_sha3_context *ctx); - -/** - * \brief This function clears a SHA-3 context. - * - * \param ctx The SHA-3 context to clear. This may be \c NULL, in which - * case this function returns immediately. If it is not \c NULL, - * it must point to an initialized SHA-3 context. - */ -void mbedtls_sha3_free(mbedtls_sha3_context *ctx); - -/** - * \brief This function clones the state of a SHA-3 context. - * - * \param dst The destination context. This must be initialized. - * \param src The context to clone. This must be initialized. - */ -void mbedtls_sha3_clone(mbedtls_sha3_context *dst, - const mbedtls_sha3_context *src); - -/** - * \brief This function starts a SHA-3 checksum - * calculation. - * - * \param ctx The context to use. This must be initialized. - * \param id The id of the SHA-3 family. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha3_starts(mbedtls_sha3_context *ctx, mbedtls_sha3_id id); - -/** - * \brief This function feeds an input buffer into an ongoing - * SHA-3 checksum calculation. - * - * \param ctx The SHA-3 context. This must be initialized - * and have a hash operation started. - * \param input The buffer holding the data. This must be a readable - * buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha3_update(mbedtls_sha3_context *ctx, - const uint8_t *input, - size_t ilen); - -/** - * \brief This function finishes the SHA-3 operation, and writes - * the result to the output buffer. - * - * \param ctx The SHA-3 context. This must be initialized - * and have a hash operation started. - * \param output The SHA-3 checksum result. - * This must be a writable buffer of length \c olen bytes. - * \param olen Defines the length of output buffer (in bytes). For SHA-3 224, SHA-3 256, - * SHA-3 384 and SHA-3 512 \c olen must equal to 28, 32, 48 and 64, - * respectively. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha3_finish(mbedtls_sha3_context *ctx, - uint8_t *output, size_t olen); - -/** - * \brief This function calculates the SHA-3 - * checksum of a buffer. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The SHA-3 result is calculated as - * output = SHA-3(id, input buffer, d). - * - * \param id The id of the SHA-3 family. - * \param input The buffer holding the data. This must be a readable - * buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * \param output The SHA-3 checksum result. - * This must be a writable buffer of length \c olen bytes. - * \param olen Defines the length of output buffer (in bytes). For SHA-3 224, SHA-3 256, - * SHA-3 384 and SHA-3 512 \c olen must equal to 28, 32, 48 and 64, - * respectively. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha3(mbedtls_sha3_id id, const uint8_t *input, - size_t ilen, - uint8_t *output, - size_t olen); - -#if defined(MBEDTLS_SELF_TEST) -/** - * \brief Checkup routine for the algorithms implemented - * by this module: SHA3-224, SHA3-256, SHA3-384, SHA3-512. - * - * \return 0 if successful, or 1 if the test failed. - */ -int mbedtls_sha3_self_test(int verbose); -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_sha3.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/sha512.h b/android/mbedtls/arm64-v8a/include/mbedtls/sha512.h deleted file mode 100644 index 1c20e4c22..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/sha512.h +++ /dev/null @@ -1,208 +0,0 @@ -/** - * \file sha512.h - * \brief This file contains SHA-384 and SHA-512 definitions and functions. - * - * The Secure Hash Algorithms 384 and 512 (SHA-384 and SHA-512) cryptographic - * hash functions are defined in FIPS 180-4: Secure Hash Standard (SHS). - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SHA512_H -#define MBEDTLS_SHA512_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include -#include - -/** SHA-512 input data was malformed. */ -#define MBEDTLS_ERR_SHA512_BAD_INPUT_DATA -0x0075 - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_SHA512_ALT) -// Regular implementation -// - -/** - * \brief The SHA-512 context structure. - * - * The structure is used both for SHA-384 and for SHA-512 - * checksum calculations. The choice between these two is - * made in the call to mbedtls_sha512_starts(). - */ -typedef struct mbedtls_sha512_context { - uint64_t MBEDTLS_PRIVATE(total)[2]; /*!< The number of Bytes processed. */ - uint64_t MBEDTLS_PRIVATE(state)[8]; /*!< The intermediate digest state. */ - unsigned char MBEDTLS_PRIVATE(buffer)[128]; /*!< The data block being processed. */ -#if defined(MBEDTLS_SHA384_C) - int MBEDTLS_PRIVATE(is384); /*!< Determines which function to use: - 0: Use SHA-512, or 1: Use SHA-384. */ -#endif -} -mbedtls_sha512_context; - -#else /* MBEDTLS_SHA512_ALT */ -#include "sha512_alt.h" -#endif /* MBEDTLS_SHA512_ALT */ - -/** - * \brief This function initializes a SHA-512 context. - * - * \param ctx The SHA-512 context to initialize. This must - * not be \c NULL. - */ -void mbedtls_sha512_init(mbedtls_sha512_context *ctx); - -/** - * \brief This function clears a SHA-512 context. - * - * \param ctx The SHA-512 context to clear. This may be \c NULL, - * in which case this function does nothing. If it - * is not \c NULL, it must point to an initialized - * SHA-512 context. - */ -void mbedtls_sha512_free(mbedtls_sha512_context *ctx); - -/** - * \brief This function clones the state of a SHA-512 context. - * - * \param dst The destination context. This must be initialized. - * \param src The context to clone. This must be initialized. - */ -void mbedtls_sha512_clone(mbedtls_sha512_context *dst, - const mbedtls_sha512_context *src); - -/** - * \brief This function starts a SHA-384 or SHA-512 checksum - * calculation. - * - * \param ctx The SHA-512 context to use. This must be initialized. - * \param is384 Determines which function to use. This must be - * either \c 0 for SHA-512, or \c 1 for SHA-384. - * - * \note is384 must be defined accordingly to the enabled - * MBEDTLS_SHA384_C/MBEDTLS_SHA512_C symbols otherwise the - * function will return #MBEDTLS_ERR_SHA512_BAD_INPUT_DATA. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha512_starts(mbedtls_sha512_context *ctx, int is384); - -/** - * \brief This function feeds an input buffer into an ongoing - * SHA-512 checksum calculation. - * - * \param ctx The SHA-512 context. This must be initialized - * and have a hash operation started. - * \param input The buffer holding the input data. This must - * be a readable buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha512_update(mbedtls_sha512_context *ctx, - const unsigned char *input, - size_t ilen); - -/** - * \brief This function finishes the SHA-512 operation, and writes - * the result to the output buffer. - * - * \param ctx The SHA-512 context. This must be initialized - * and have a hash operation started. - * \param output The SHA-384 or SHA-512 checksum result. - * This must be a writable buffer of length \c 64 bytes - * for SHA-512, \c 48 bytes for SHA-384. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha512_finish(mbedtls_sha512_context *ctx, - unsigned char *output); - -/** - * \brief This function processes a single data block within - * the ongoing SHA-512 computation. - * This function is for internal use only. - * - * \param ctx The SHA-512 context. This must be initialized. - * \param data The buffer holding one block of data. This - * must be a readable buffer of length \c 128 Bytes. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_internal_sha512_process(mbedtls_sha512_context *ctx, - const unsigned char data[128]); - -/** - * \brief This function calculates the SHA-512 or SHA-384 - * checksum of a buffer. - * - * The function allocates the context, performs the - * calculation, and frees the context. - * - * The SHA-512 result is calculated as - * output = SHA-512(input buffer). - * - * \param input The buffer holding the input data. This must be - * a readable buffer of length \p ilen Bytes. - * \param ilen The length of the input data in Bytes. - * \param output The SHA-384 or SHA-512 checksum result. - * This must be a writable buffer of length \c 64 bytes - * for SHA-512, \c 48 bytes for SHA-384. - * \param is384 Determines which function to use. This must be either - * \c 0 for SHA-512, or \c 1 for SHA-384. - * - * \note is384 must be defined accordingly with the supported - * symbols in the config file. If: - * - is384 is 0, but \c MBEDTLS_SHA384_C is not defined, or - * - is384 is 1, but \c MBEDTLS_SHA512_C is not defined - * then the function will return - * #MBEDTLS_ERR_SHA512_BAD_INPUT_DATA. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_sha512(const unsigned char *input, - size_t ilen, - unsigned char *output, - int is384); - -#if defined(MBEDTLS_SELF_TEST) - -#if defined(MBEDTLS_SHA384_C) -/** - * \brief The SHA-384 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_sha384_self_test(int verbose); -#endif /* MBEDTLS_SHA384_C */ - -#if defined(MBEDTLS_SHA512_C) -/** - * \brief The SHA-512 checkup routine. - * - * \return \c 0 on success. - * \return \c 1 on failure. - */ -int mbedtls_sha512_self_test(int verbose); -#endif /* MBEDTLS_SHA512_C */ - -#endif /* MBEDTLS_SELF_TEST */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_sha512.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ssl.h b/android/mbedtls/arm64-v8a/include/mbedtls/ssl.h deleted file mode 100644 index 3cdddf7d7..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ssl.h +++ /dev/null @@ -1,5821 +0,0 @@ -/** - * \file ssl.h - * - * \brief SSL/TLS functions. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SSL_H -#define MBEDTLS_SSL_H -#include "mbedtls/platform_util.h" -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/bignum.h" -#include "mbedtls/ecp.h" - -#include "mbedtls/ssl_ciphersuites.h" - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -#include "mbedtls/x509_crt.h" -#include "mbedtls/x509_crl.h" -#endif - -#if defined(MBEDTLS_DHM_C) -#include "mbedtls/dhm.h" -#endif - -#include "mbedtls/md.h" - -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_ANY_ENABLED) -#include "mbedtls/ecdh.h" -#endif - -#if defined(MBEDTLS_HAVE_TIME) -#include "mbedtls/platform_time.h" -#endif - -#include "psa/crypto.h" - -/* - * SSL Error codes - */ -/** A cryptographic operation is in progress. Try again later. */ -#define MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS -0x7000 -/** The requested feature is not available. */ -#define MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE -0x7080 -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_SSL_BAD_INPUT_DATA -0x7100 -/** Verification of the message MAC failed. */ -#define MBEDTLS_ERR_SSL_INVALID_MAC -0x7180 -/** An invalid SSL record was received. */ -#define MBEDTLS_ERR_SSL_INVALID_RECORD -0x7200 -/** The connection indicated an EOF. */ -#define MBEDTLS_ERR_SSL_CONN_EOF -0x7280 -/** A message could not be parsed due to a syntactic error. */ -#define MBEDTLS_ERR_SSL_DECODE_ERROR -0x7300 -/* Error space gap */ -/** No RNG was provided to the SSL module. */ -#define MBEDTLS_ERR_SSL_NO_RNG -0x7400 -/** No client certification received from the client, but required by the authentication mode. */ -#define MBEDTLS_ERR_SSL_NO_CLIENT_CERTIFICATE -0x7480 -/** Client received an extended server hello containing an unsupported extension */ -#define MBEDTLS_ERR_SSL_UNSUPPORTED_EXTENSION -0x7500 -/** No ALPN protocols supported that the client advertises */ -#define MBEDTLS_ERR_SSL_NO_APPLICATION_PROTOCOL -0x7580 -/** The own private key or pre-shared key is not set, but needed. */ -#define MBEDTLS_ERR_SSL_PRIVATE_KEY_REQUIRED -0x7600 -/** No CA Chain is set, but required to operate. */ -#define MBEDTLS_ERR_SSL_CA_CHAIN_REQUIRED -0x7680 -/** An unexpected message was received from our peer. */ -#define MBEDTLS_ERR_SSL_UNEXPECTED_MESSAGE -0x7700 -/** A fatal alert message was received from our peer. */ -#define MBEDTLS_ERR_SSL_FATAL_ALERT_MESSAGE -0x7780 -/** No server could be identified matching the client's SNI. */ -#define MBEDTLS_ERR_SSL_UNRECOGNIZED_NAME -0x7800 -/** The peer notified us that the connection is going to be closed. */ -#define MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY -0x7880 -/* Error space gap */ -/* Error space gap */ -/** Processing of the Certificate handshake message failed. */ -#define MBEDTLS_ERR_SSL_BAD_CERTIFICATE -0x7A00 -/* Error space gap */ -/** A TLS 1.3 NewSessionTicket message has been received. */ -#define MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET -0x7B00 -/** Not possible to read early data */ -#define MBEDTLS_ERR_SSL_CANNOT_READ_EARLY_DATA -0x7B80 -/** - * Early data has been received as part of an on-going handshake. - * This error code can be returned only on server side if and only if early - * data has been enabled by means of the mbedtls_ssl_conf_early_data() API. - * This error code can then be returned by mbedtls_ssl_handshake(), - * mbedtls_ssl_handshake_step(), mbedtls_ssl_read() or mbedtls_ssl_write() if - * early data has been received as part of the handshake sequence they - * triggered. To read the early data, call mbedtls_ssl_read_early_data(). - */ -#define MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA -0x7C00 -/** Not possible to write early data */ -#define MBEDTLS_ERR_SSL_CANNOT_WRITE_EARLY_DATA -0x7C80 -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/** Cache entry not found */ -#define MBEDTLS_ERR_SSL_CACHE_ENTRY_NOT_FOUND -0x7E80 -/** Memory allocation failed */ -#define MBEDTLS_ERR_SSL_ALLOC_FAILED -0x7F00 -/** Hardware acceleration function returned with error */ -#define MBEDTLS_ERR_SSL_HW_ACCEL_FAILED -0x7F80 -/** Hardware acceleration function skipped / left alone data */ -#define MBEDTLS_ERR_SSL_HW_ACCEL_FALLTHROUGH -0x6F80 -/** Handshake protocol not within min/max boundaries */ -#define MBEDTLS_ERR_SSL_BAD_PROTOCOL_VERSION -0x6E80 -/** The handshake negotiation failed. */ -#define MBEDTLS_ERR_SSL_HANDSHAKE_FAILURE -0x6E00 -/** Session ticket has expired. */ -#define MBEDTLS_ERR_SSL_SESSION_TICKET_EXPIRED -0x6D80 -/** Public key type mismatch (eg, asked for RSA key exchange and presented EC key) */ -#define MBEDTLS_ERR_SSL_PK_TYPE_MISMATCH -0x6D00 -/** Unknown identity received (eg, PSK identity) */ -#define MBEDTLS_ERR_SSL_UNKNOWN_IDENTITY -0x6C80 -/** Internal error (eg, unexpected failure in lower-level module) */ -#define MBEDTLS_ERR_SSL_INTERNAL_ERROR -0x6C00 -/** A counter would wrap (eg, too many messages exchanged). */ -#define MBEDTLS_ERR_SSL_COUNTER_WRAPPING -0x6B80 -/** Unexpected message at ServerHello in renegotiation. */ -#define MBEDTLS_ERR_SSL_WAITING_SERVER_HELLO_RENEGO -0x6B00 -/** DTLS client must retry for hello verification */ -#define MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED -0x6A80 -/** A buffer is too small to receive or write a message */ -#define MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL -0x6A00 -/* Error space gap */ -/** No data of requested type currently available on underlying transport. */ -#define MBEDTLS_ERR_SSL_WANT_READ -0x6900 -/** Connection requires a write call. */ -#define MBEDTLS_ERR_SSL_WANT_WRITE -0x6880 -/** The operation timed out. */ -#define MBEDTLS_ERR_SSL_TIMEOUT -0x6800 -/** The client initiated a reconnect from the same port. */ -#define MBEDTLS_ERR_SSL_CLIENT_RECONNECT -0x6780 -/** Record header looks valid but is not expected. */ -#define MBEDTLS_ERR_SSL_UNEXPECTED_RECORD -0x6700 -/** The alert message received indicates a non-fatal error. */ -#define MBEDTLS_ERR_SSL_NON_FATAL -0x6680 -/** A field in a message was incorrect or inconsistent with other fields. */ -#define MBEDTLS_ERR_SSL_ILLEGAL_PARAMETER -0x6600 -/** Internal-only message signaling that further message-processing should be done */ -#define MBEDTLS_ERR_SSL_CONTINUE_PROCESSING -0x6580 -/** The asynchronous operation is not completed yet. */ -#define MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS -0x6500 -/** Internal-only message signaling that a message arrived early. */ -#define MBEDTLS_ERR_SSL_EARLY_MESSAGE -0x6480 -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/* Error space gap */ -/** An encrypted DTLS-frame with an unexpected CID was received. */ -#define MBEDTLS_ERR_SSL_UNEXPECTED_CID -0x6000 -/** An operation failed due to an unexpected version or configuration. */ -#define MBEDTLS_ERR_SSL_VERSION_MISMATCH -0x5F00 -/** Invalid value in SSL config */ -#define MBEDTLS_ERR_SSL_BAD_CONFIG -0x5E80 -/* Error space gap */ -/** Attempt to verify a certificate without an expected hostname. - * This is usually insecure. - * - * In TLS clients, when a client authenticates a server through its - * certificate, the client normally checks three things: - * - the certificate chain must be valid; - * - the chain must start from a trusted CA; - * - the certificate must cover the server name that is expected by the client. - * - * Omitting any of these checks is generally insecure, and can allow a - * malicious server to impersonate a legitimate server. - * - * The third check may be safely skipped in some unusual scenarios, - * such as networks where eavesdropping is a risk but not active attacks, - * or a private PKI where the client equally trusts all servers that are - * accredited by the root CA. - * - * You should call mbedtls_ssl_set_hostname() with the expected server name - * before starting a TLS handshake on a client (unless the client is - * set up to only use PSK-based authentication, which does not rely on the - * host name). If you have determined that server name verification is not - * required for security in your scenario, call mbedtls_ssl_set_hostname() - * with \p NULL as the server name. - * - * This error is raised if all of the following conditions are met: - * - * - A TLS client is configured with the authentication mode - * #MBEDTLS_SSL_VERIFY_REQUIRED (default). - * - Certificate authentication is enabled. - * - The client does not call mbedtls_ssl_set_hostname(). - * - The configuration option - * #MBEDTLS_SSL_CLI_ALLOW_WEAK_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * is not enabled. - */ -#define MBEDTLS_ERR_SSL_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME -0x5D80 - -/* - * Constants from RFC 8446 for TLS 1.3 PSK modes - * - * Those are used in the Pre-Shared Key Exchange Modes extension. - * See Section 4.2.9 in RFC 8446. - */ -#define MBEDTLS_SSL_TLS1_3_PSK_MODE_PURE 0 /* Pure PSK-based exchange */ -#define MBEDTLS_SSL_TLS1_3_PSK_MODE_ECDHE 1 /* PSK+ECDHE-based exchange */ - -/* - * TLS 1.3 NamedGroup values - * - * From RF 8446 - * enum { - * // Elliptic Curve Groups (ECDHE) - * secp256r1(0x0017), secp384r1(0x0018), secp521r1(0x0019), - * x25519(0x001D), x448(0x001E), - * // Finite Field Groups (DHE) - * ffdhe2048(0x0100), ffdhe3072(0x0101), ffdhe4096(0x0102), - * ffdhe6144(0x0103), ffdhe8192(0x0104), - * // Reserved Code Points - * ffdhe_private_use(0x01FC..0x01FF), - * ecdhe_private_use(0xFE00..0xFEFF), - * (0xFFFF) - * } NamedGroup; - * - */ - -/* Elliptic Curve Groups (ECDHE) */ -#define MBEDTLS_SSL_IANA_TLS_GROUP_NONE 0 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP192K1 0x0012 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP192R1 0x0013 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP224K1 0x0014 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP224R1 0x0015 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP256K1 0x0016 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP256R1 0x0017 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP384R1 0x0018 -#define MBEDTLS_SSL_IANA_TLS_GROUP_SECP521R1 0x0019 -#define MBEDTLS_SSL_IANA_TLS_GROUP_BP256R1 0x001A -#define MBEDTLS_SSL_IANA_TLS_GROUP_BP384R1 0x001B -#define MBEDTLS_SSL_IANA_TLS_GROUP_BP512R1 0x001C -#define MBEDTLS_SSL_IANA_TLS_GROUP_X25519 0x001D -#define MBEDTLS_SSL_IANA_TLS_GROUP_X448 0x001E -/* Finite Field Groups (DHE) */ -#define MBEDTLS_SSL_IANA_TLS_GROUP_FFDHE2048 0x0100 -#define MBEDTLS_SSL_IANA_TLS_GROUP_FFDHE3072 0x0101 -#define MBEDTLS_SSL_IANA_TLS_GROUP_FFDHE4096 0x0102 -#define MBEDTLS_SSL_IANA_TLS_GROUP_FFDHE6144 0x0103 -#define MBEDTLS_SSL_IANA_TLS_GROUP_FFDHE8192 0x0104 - -/* - * TLS 1.3 Key Exchange Modes - * - * Mbed TLS internal identifiers for use with the SSL configuration API - * mbedtls_ssl_conf_tls13_key_exchange_modes(). - */ - -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK (1u << 0) /*!< Pure-PSK TLS 1.3 key exchange, - * encompassing both externally agreed PSKs - * as well as resumption PSKs. */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL (1u << 1) /*!< Pure-Ephemeral TLS 1.3 key exchanges, - * including for example ECDHE and DHE - * key exchanges. */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL (1u << 2) /*!< PSK-Ephemeral TLS 1.3 key exchanges, - * using both a PSK and an ephemeral - * key exchange. */ - -/* Convenience macros for sets of key exchanges. */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_ALL \ - (MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK | \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL | \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL) /*!< All TLS 1.3 key exchanges */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ALL \ - (MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK | \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL) /*!< All PSK-based TLS 1.3 key exchanges */ -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ALL \ - (MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL | \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL) /*!< All ephemeral TLS 1.3 key exchanges */ - -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_NONE (0) - -/* - * Various constants - */ - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/* These are the high and low bytes of ProtocolVersion as defined by: - * - RFC 5246: ProtocolVersion version = { 3, 3 }; // TLS v1.2 - * - RFC 8446: see section 4.2.1 - */ -#define MBEDTLS_SSL_MAJOR_VERSION_3 3 -#define MBEDTLS_SSL_MINOR_VERSION_3 3 /*!< TLS v1.2 */ -#define MBEDTLS_SSL_MINOR_VERSION_4 4 /*!< TLS v1.3 */ -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -#define MBEDTLS_SSL_TRANSPORT_STREAM 0 /*!< TLS */ -#define MBEDTLS_SSL_TRANSPORT_DATAGRAM 1 /*!< DTLS */ - -#define MBEDTLS_SSL_MAX_HOST_NAME_LEN 255 /*!< Maximum host name defined in RFC 1035 */ -#define MBEDTLS_SSL_MAX_ALPN_NAME_LEN 255 /*!< Maximum size in bytes of a protocol name in alpn ext., RFC 7301 */ - -#define MBEDTLS_SSL_MAX_ALPN_LIST_LEN 65535 /*!< Maximum size in bytes of list in alpn ext., RFC 7301 */ - -/* RFC 6066 section 4, see also mfl_code_to_length in ssl_tls.c - * NONE must be zero so that memset()ing structure to zero works */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_NONE 0 /*!< don't use this extension */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_512 1 /*!< MaxFragmentLength 2^9 */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_1024 2 /*!< MaxFragmentLength 2^10 */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_2048 3 /*!< MaxFragmentLength 2^11 */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_4096 4 /*!< MaxFragmentLength 2^12 */ -#define MBEDTLS_SSL_MAX_FRAG_LEN_INVALID 5 /*!< first invalid value */ - -#define MBEDTLS_SSL_IS_CLIENT 0 -#define MBEDTLS_SSL_IS_SERVER 1 - -#define MBEDTLS_SSL_EXTENDED_MS_DISABLED 0 -#define MBEDTLS_SSL_EXTENDED_MS_ENABLED 1 - -#define MBEDTLS_SSL_CID_DISABLED 0 -#define MBEDTLS_SSL_CID_ENABLED 1 - -#define MBEDTLS_SSL_ETM_DISABLED 0 -#define MBEDTLS_SSL_ETM_ENABLED 1 - -#define MBEDTLS_SSL_COMPRESS_NULL 0 - -#define MBEDTLS_SSL_VERIFY_NONE 0 -#define MBEDTLS_SSL_VERIFY_OPTIONAL 1 -#define MBEDTLS_SSL_VERIFY_REQUIRED 2 -#define MBEDTLS_SSL_VERIFY_UNSET 3 /* Used only for sni_authmode */ - -#define MBEDTLS_SSL_LEGACY_RENEGOTIATION 0 -#define MBEDTLS_SSL_SECURE_RENEGOTIATION 1 - -#define MBEDTLS_SSL_RENEGOTIATION_DISABLED 0 -#define MBEDTLS_SSL_RENEGOTIATION_ENABLED 1 - -#define MBEDTLS_SSL_ANTI_REPLAY_DISABLED 0 -#define MBEDTLS_SSL_ANTI_REPLAY_ENABLED 1 - -#define MBEDTLS_SSL_RENEGOTIATION_NOT_ENFORCED -1 -#define MBEDTLS_SSL_RENEGO_MAX_RECORDS_DEFAULT 16 - -#define MBEDTLS_SSL_LEGACY_NO_RENEGOTIATION 0 -#define MBEDTLS_SSL_LEGACY_ALLOW_RENEGOTIATION 1 -#define MBEDTLS_SSL_LEGACY_BREAK_HANDSHAKE 2 - -#define MBEDTLS_SSL_TRUNC_HMAC_DISABLED 0 -#define MBEDTLS_SSL_TRUNC_HMAC_ENABLED 1 -#define MBEDTLS_SSL_TRUNCATED_HMAC_LEN 10 /* 80 bits, rfc 6066 section 7 */ - -#define MBEDTLS_SSL_SESSION_TICKETS_DISABLED 0 -#define MBEDTLS_SSL_SESSION_TICKETS_ENABLED 1 - -#define MBEDTLS_SSL_TLS1_3_SIGNAL_NEW_SESSION_TICKETS_DISABLED 0 -#define MBEDTLS_SSL_TLS1_3_SIGNAL_NEW_SESSION_TICKETS_ENABLED 1 - -#define MBEDTLS_SSL_PRESET_DEFAULT 0 -#define MBEDTLS_SSL_PRESET_SUITEB 2 - -#define MBEDTLS_SSL_CERT_REQ_CA_LIST_ENABLED 1 -#define MBEDTLS_SSL_CERT_REQ_CA_LIST_DISABLED 0 - -#define MBEDTLS_SSL_EARLY_DATA_DISABLED 0 -#define MBEDTLS_SSL_EARLY_DATA_ENABLED 1 - -#define MBEDTLS_SSL_DTLS_SRTP_MKI_UNSUPPORTED 0 -#define MBEDTLS_SSL_DTLS_SRTP_MKI_SUPPORTED 1 - -#define MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_CLIENT 1 -#define MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_SERVER 0 - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && defined(MBEDTLS_SSL_SESSION_TICKETS) -#if defined(PSA_WANT_ALG_SHA_384) -#define MBEDTLS_SSL_TLS1_3_TICKET_RESUMPTION_KEY_LEN 48 -#elif defined(PSA_WANT_ALG_SHA_256) -#define MBEDTLS_SSL_TLS1_3_TICKET_RESUMPTION_KEY_LEN 32 -#endif -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 && MBEDTLS_SSL_SESSION_TICKETS */ -/* - * Default range for DTLS retransmission timer value, in milliseconds. - * RFC 6347 4.2.4.1 says from 1 second to 60 seconds. - */ -#define MBEDTLS_SSL_DTLS_TIMEOUT_DFL_MIN 1000 -#define MBEDTLS_SSL_DTLS_TIMEOUT_DFL_MAX 60000 - -/* - * Whether early data record should be discarded or not and how. - * - * The client has indicated early data and the server has rejected them. - * The server has then to skip past early data by either: - * - attempting to deprotect received records using the handshake traffic - * key, discarding records which fail deprotection (up to the configured - * max_early_data_size). Once a record is deprotected successfully, - * it is treated as the start of the client's second flight and the - * server proceeds as with an ordinary 1-RTT handshake. - * - skipping all records with an external content type of - * "application_data" (indicating that they are encrypted), up to the - * configured max_early_data_size. This is the expected behavior if the - * server has sent an HelloRetryRequest message. The server ignores - * application data message before 2nd ClientHello. - */ -#define MBEDTLS_SSL_EARLY_DATA_NO_DISCARD 0 -#define MBEDTLS_SSL_EARLY_DATA_TRY_TO_DEPROTECT_AND_DISCARD 1 -#define MBEDTLS_SSL_EARLY_DATA_DISCARD 2 - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -/* - * Maximum fragment length in bytes, - * determines the size of each of the two internal I/O buffers. - * - * Note: the RFC defines the default size of SSL / TLS messages. If you - * change the value here, other clients / servers may not be able to - * communicate with you anymore. Only change this value if you control - * both sides of the connection and have it reduced at both sides, or - * if you're using the Max Fragment Length extension and you know all your - * peers are using it too! - */ -#if !defined(MBEDTLS_SSL_IN_CONTENT_LEN) -#define MBEDTLS_SSL_IN_CONTENT_LEN 16384 -#endif - -#if !defined(MBEDTLS_SSL_OUT_CONTENT_LEN) -#define MBEDTLS_SSL_OUT_CONTENT_LEN 16384 -#endif - -/* - * Maximum number of heap-allocated bytes for the purpose of - * DTLS handshake message reassembly and future message buffering. - */ -#if !defined(MBEDTLS_SSL_DTLS_MAX_BUFFERING) -#define MBEDTLS_SSL_DTLS_MAX_BUFFERING 32768 -#endif - -/* - * Maximum length of CIDs for incoming and outgoing messages. - */ -#if !defined(MBEDTLS_SSL_CID_IN_LEN_MAX) -#define MBEDTLS_SSL_CID_IN_LEN_MAX 32 -#endif - -#if !defined(MBEDTLS_SSL_CID_OUT_LEN_MAX) -#define MBEDTLS_SSL_CID_OUT_LEN_MAX 32 -#endif - -#if !defined(MBEDTLS_SSL_CID_TLS1_3_PADDING_GRANULARITY) -#define MBEDTLS_SSL_CID_TLS1_3_PADDING_GRANULARITY 16 -#endif - -#if !defined(MBEDTLS_SSL_MAX_EARLY_DATA_SIZE) -#define MBEDTLS_SSL_MAX_EARLY_DATA_SIZE 1024 -#endif - -#if !defined(MBEDTLS_SSL_TLS1_3_TICKET_AGE_TOLERANCE) -#define MBEDTLS_SSL_TLS1_3_TICKET_AGE_TOLERANCE 6000 -#endif - -#if !defined(MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH) -#define MBEDTLS_SSL_TLS1_3_TICKET_NONCE_LENGTH 32 -#endif - -#if !defined(MBEDTLS_SSL_TLS1_3_DEFAULT_NEW_SESSION_TICKETS) -#define MBEDTLS_SSL_TLS1_3_DEFAULT_NEW_SESSION_TICKETS 1 -#endif - -/** \} name SECTION: Module settings */ - -/* - * Default to standard CID mode - */ -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) && \ - !defined(MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT) -#define MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT 0 -#endif - -/* - * Length of the verify data for secure renegotiation - */ -#define MBEDTLS_SSL_VERIFY_DATA_MAX_LEN 12 - -/* - * Signaling ciphersuite values (SCSV) - */ -#define MBEDTLS_SSL_EMPTY_RENEGOTIATION_INFO 0xFF /**< renegotiation info ext */ - -/* - * Supported Signature and Hash algorithms (For TLS 1.2) - * RFC 5246 section 7.4.1.4.1 - */ -#define MBEDTLS_SSL_HASH_NONE 0 -#define MBEDTLS_SSL_HASH_MD5 1 -#define MBEDTLS_SSL_HASH_SHA1 2 -#define MBEDTLS_SSL_HASH_SHA224 3 -#define MBEDTLS_SSL_HASH_SHA256 4 -#define MBEDTLS_SSL_HASH_SHA384 5 -#define MBEDTLS_SSL_HASH_SHA512 6 - -#define MBEDTLS_SSL_SIG_ANON 0 -#define MBEDTLS_SSL_SIG_RSA 1 -#define MBEDTLS_SSL_SIG_ECDSA 3 - -/* - * TLS 1.3 signature algorithms - * RFC 8446, Section 4.2.3 - */ - -/* RSASSA-PKCS1-v1_5 algorithms */ -#define MBEDTLS_TLS1_3_SIG_RSA_PKCS1_SHA256 0x0401 -#define MBEDTLS_TLS1_3_SIG_RSA_PKCS1_SHA384 0x0501 -#define MBEDTLS_TLS1_3_SIG_RSA_PKCS1_SHA512 0x0601 - -/* ECDSA algorithms */ -#define MBEDTLS_TLS1_3_SIG_ECDSA_SECP256R1_SHA256 0x0403 -#define MBEDTLS_TLS1_3_SIG_ECDSA_SECP384R1_SHA384 0x0503 -#define MBEDTLS_TLS1_3_SIG_ECDSA_SECP521R1_SHA512 0x0603 - -/* RSASSA-PSS algorithms with public key OID rsaEncryption */ -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA256 0x0804 -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA384 0x0805 -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_RSAE_SHA512 0x0806 - -/* EdDSA algorithms */ -#define MBEDTLS_TLS1_3_SIG_ED25519 0x0807 -#define MBEDTLS_TLS1_3_SIG_ED448 0x0808 - -/* RSASSA-PSS algorithms with public key OID RSASSA-PSS */ -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_PSS_SHA256 0x0809 -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_PSS_SHA384 0x080A -#define MBEDTLS_TLS1_3_SIG_RSA_PSS_PSS_SHA512 0x080B - -/* LEGACY ALGORITHMS */ -#define MBEDTLS_TLS1_3_SIG_RSA_PKCS1_SHA1 0x0201 -#define MBEDTLS_TLS1_3_SIG_ECDSA_SHA1 0x0203 - -#define MBEDTLS_TLS1_3_SIG_NONE 0x0 - -/* - * Client Certificate Types - * RFC 5246 section 7.4.4 plus RFC 4492 section 5.5 - */ -#define MBEDTLS_SSL_CERT_TYPE_RSA_SIGN 1 -#define MBEDTLS_SSL_CERT_TYPE_ECDSA_SIGN 64 - -/* - * Message, alert and handshake types - */ -#define MBEDTLS_SSL_MSG_CHANGE_CIPHER_SPEC 20 -#define MBEDTLS_SSL_MSG_ALERT 21 -#define MBEDTLS_SSL_MSG_HANDSHAKE 22 -#define MBEDTLS_SSL_MSG_APPLICATION_DATA 23 -#define MBEDTLS_SSL_MSG_CID 25 - -#define MBEDTLS_SSL_ALERT_LEVEL_WARNING 1 -#define MBEDTLS_SSL_ALERT_LEVEL_FATAL 2 - -#define MBEDTLS_SSL_ALERT_MSG_CLOSE_NOTIFY 0 /* 0x00 */ -#define MBEDTLS_SSL_ALERT_MSG_UNEXPECTED_MESSAGE 10 /* 0x0A */ -#define MBEDTLS_SSL_ALERT_MSG_BAD_RECORD_MAC 20 /* 0x14 */ -#define MBEDTLS_SSL_ALERT_MSG_DECRYPTION_FAILED 21 /* 0x15 */ -#define MBEDTLS_SSL_ALERT_MSG_RECORD_OVERFLOW 22 /* 0x16 */ -#define MBEDTLS_SSL_ALERT_MSG_DECOMPRESSION_FAILURE 30 /* 0x1E */ -#define MBEDTLS_SSL_ALERT_MSG_HANDSHAKE_FAILURE 40 /* 0x28 */ -#define MBEDTLS_SSL_ALERT_MSG_NO_CERT 41 /* 0x29 */ -#define MBEDTLS_SSL_ALERT_MSG_BAD_CERT 42 /* 0x2A */ -#define MBEDTLS_SSL_ALERT_MSG_UNSUPPORTED_CERT 43 /* 0x2B */ -#define MBEDTLS_SSL_ALERT_MSG_CERT_REVOKED 44 /* 0x2C */ -#define MBEDTLS_SSL_ALERT_MSG_CERT_EXPIRED 45 /* 0x2D */ -#define MBEDTLS_SSL_ALERT_MSG_CERT_UNKNOWN 46 /* 0x2E */ -#define MBEDTLS_SSL_ALERT_MSG_ILLEGAL_PARAMETER 47 /* 0x2F */ -#define MBEDTLS_SSL_ALERT_MSG_UNKNOWN_CA 48 /* 0x30 */ -#define MBEDTLS_SSL_ALERT_MSG_ACCESS_DENIED 49 /* 0x31 */ -#define MBEDTLS_SSL_ALERT_MSG_DECODE_ERROR 50 /* 0x32 */ -#define MBEDTLS_SSL_ALERT_MSG_DECRYPT_ERROR 51 /* 0x33 */ -#define MBEDTLS_SSL_ALERT_MSG_EXPORT_RESTRICTION 60 /* 0x3C */ -#define MBEDTLS_SSL_ALERT_MSG_PROTOCOL_VERSION 70 /* 0x46 */ -#define MBEDTLS_SSL_ALERT_MSG_INSUFFICIENT_SECURITY 71 /* 0x47 */ -#define MBEDTLS_SSL_ALERT_MSG_INTERNAL_ERROR 80 /* 0x50 */ -#define MBEDTLS_SSL_ALERT_MSG_INAPROPRIATE_FALLBACK 86 /* 0x56 */ -#define MBEDTLS_SSL_ALERT_MSG_USER_CANCELED 90 /* 0x5A */ -#define MBEDTLS_SSL_ALERT_MSG_NO_RENEGOTIATION 100 /* 0x64 */ -#define MBEDTLS_SSL_ALERT_MSG_MISSING_EXTENSION 109 /* 0x6d -- new in TLS 1.3 */ -#define MBEDTLS_SSL_ALERT_MSG_UNSUPPORTED_EXT 110 /* 0x6E */ -#define MBEDTLS_SSL_ALERT_MSG_UNRECOGNIZED_NAME 112 /* 0x70 */ -#define MBEDTLS_SSL_ALERT_MSG_UNKNOWN_PSK_IDENTITY 115 /* 0x73 */ -#define MBEDTLS_SSL_ALERT_MSG_CERT_REQUIRED 116 /* 0x74 */ -#define MBEDTLS_SSL_ALERT_MSG_NO_APPLICATION_PROTOCOL 120 /* 0x78 */ - -#define MBEDTLS_SSL_HS_HELLO_REQUEST 0 -#define MBEDTLS_SSL_HS_CLIENT_HELLO 1 -#define MBEDTLS_SSL_HS_SERVER_HELLO 2 -#define MBEDTLS_SSL_HS_HELLO_VERIFY_REQUEST 3 -#define MBEDTLS_SSL_HS_NEW_SESSION_TICKET 4 -#define MBEDTLS_SSL_HS_END_OF_EARLY_DATA 5 -#define MBEDTLS_SSL_HS_ENCRYPTED_EXTENSIONS 8 -#define MBEDTLS_SSL_HS_CERTIFICATE 11 -#define MBEDTLS_SSL_HS_SERVER_KEY_EXCHANGE 12 -#define MBEDTLS_SSL_HS_CERTIFICATE_REQUEST 13 -#define MBEDTLS_SSL_HS_SERVER_HELLO_DONE 14 -#define MBEDTLS_SSL_HS_CERTIFICATE_VERIFY 15 -#define MBEDTLS_SSL_HS_CLIENT_KEY_EXCHANGE 16 -#define MBEDTLS_SSL_HS_FINISHED 20 -#define MBEDTLS_SSL_HS_MESSAGE_HASH 254 - -/* - * TLS extensions - */ -#define MBEDTLS_TLS_EXT_SERVERNAME 0 -#define MBEDTLS_TLS_EXT_SERVERNAME_HOSTNAME 0 - -#define MBEDTLS_TLS_EXT_MAX_FRAGMENT_LENGTH 1 - -#define MBEDTLS_TLS_EXT_TRUNCATED_HMAC 4 -#define MBEDTLS_TLS_EXT_STATUS_REQUEST 5 /* RFC 6066 TLS 1.2 and 1.3 */ - -#define MBEDTLS_TLS_EXT_SUPPORTED_ELLIPTIC_CURVES 10 -#define MBEDTLS_TLS_EXT_SUPPORTED_GROUPS 10 /* RFC 8422,7919 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_SUPPORTED_POINT_FORMATS 11 - -#define MBEDTLS_TLS_EXT_SIG_ALG 13 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_USE_SRTP 14 -#define MBEDTLS_TLS_EXT_HEARTBEAT 15 /* RFC 6520 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_ALPN 16 - -#define MBEDTLS_TLS_EXT_SCT 18 /* RFC 6962 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_CLI_CERT_TYPE 19 /* RFC 7250 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_SERV_CERT_TYPE 20 /* RFC 7250 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_PADDING 21 /* RFC 7685 TLS 1.2 and 1.3 */ -#define MBEDTLS_TLS_EXT_ENCRYPT_THEN_MAC 22 /* 0x16 */ -#define MBEDTLS_TLS_EXT_EXTENDED_MASTER_SECRET 0x0017 /* 23 */ - -#define MBEDTLS_TLS_EXT_RECORD_SIZE_LIMIT 28 /* RFC 8449 (implemented for TLS 1.3 only) */ - -#define MBEDTLS_TLS_EXT_SESSION_TICKET 35 - -#define MBEDTLS_TLS_EXT_PRE_SHARED_KEY 41 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_EARLY_DATA 42 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_SUPPORTED_VERSIONS 43 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_COOKIE 44 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_PSK_KEY_EXCHANGE_MODES 45 /* RFC 8446 TLS 1.3 */ - -#define MBEDTLS_TLS_EXT_CERT_AUTH 47 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_OID_FILTERS 48 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_POST_HANDSHAKE_AUTH 49 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_SIG_ALG_CERT 50 /* RFC 8446 TLS 1.3 */ -#define MBEDTLS_TLS_EXT_KEY_SHARE 51 /* RFC 8446 TLS 1.3 */ - -#if MBEDTLS_SSL_DTLS_CONNECTION_ID_COMPAT == 0 -#define MBEDTLS_TLS_EXT_CID 54 /* RFC 9146 DTLS 1.2 CID */ -#else -#define MBEDTLS_TLS_EXT_CID 254 /* Pre-RFC 9146 DTLS 1.2 CID */ -#endif - -#define MBEDTLS_TLS_EXT_ECJPAKE_KKPP 256 /* experimental */ - -#define MBEDTLS_TLS_EXT_RENEGOTIATION_INFO 0xFF01 - -/* - * Size defines - */ -#if !defined(MBEDTLS_PSK_MAX_LEN) -/* - * If the library supports TLS 1.3 tickets and the cipher suite - * TLS1-3-AES-256-GCM-SHA384, set the PSK maximum length to 48 instead of 32. - * That way, the TLS 1.3 client and server are able to resume sessions where - * the cipher suite is TLS1-3-AES-256-GCM-SHA384 (pre-shared keys are 48 - * bytes long in that case). - */ -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && \ - defined(MBEDTLS_SSL_SESSION_TICKETS) && \ - defined(MBEDTLS_SSL_HAVE_AES) && defined(MBEDTLS_SSL_HAVE_GCM) && \ - defined(MBEDTLS_MD_CAN_SHA384) -#define MBEDTLS_PSK_MAX_LEN 48 /* 384 bits */ -#else -#define MBEDTLS_PSK_MAX_LEN 32 /* 256 bits */ -#endif -#endif /* !MBEDTLS_PSK_MAX_LEN */ - -/* Dummy type used only for its size */ -union mbedtls_ssl_premaster_secret { - unsigned char dummy; /* Make the union non-empty even with SSL disabled */ -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) - unsigned char _pms_rsa[48]; /* RFC 5246 8.1.1 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) - unsigned char _pms_dhm[MBEDTLS_MPI_MAX_SIZE]; /* RFC 5246 8.1.2 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) - unsigned char _pms_ecdh[MBEDTLS_ECP_MAX_BYTES]; /* RFC 4492 5.10 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_PSK_ENABLED) - unsigned char _pms_psk[4 + 2 * MBEDTLS_PSK_MAX_LEN]; /* RFC 4279 2 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) - unsigned char _pms_dhe_psk[4 + MBEDTLS_MPI_MAX_SIZE - + MBEDTLS_PSK_MAX_LEN]; /* RFC 4279 3 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) - unsigned char _pms_rsa_psk[52 + MBEDTLS_PSK_MAX_LEN]; /* RFC 4279 4 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) - unsigned char _pms_ecdhe_psk[4 + MBEDTLS_ECP_MAX_BYTES - + MBEDTLS_PSK_MAX_LEN]; /* RFC 5489 2 */ -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) - unsigned char _pms_ecjpake[32]; /* Thread spec: SHA-256 output */ -#endif -}; - -#define MBEDTLS_PREMASTER_SIZE sizeof(union mbedtls_ssl_premaster_secret) - -#define MBEDTLS_TLS1_3_MD_MAX_SIZE PSA_HASH_MAX_SIZE - - -/* Length in number of bytes of the TLS sequence number */ -#define MBEDTLS_SSL_SEQUENCE_NUMBER_LEN 8 - -/* Helper to state that client_random and server_random need to be stored - * after the handshake is complete. This is required for context serialization - * and for the keying material exporter in TLS 1.2. */ -#if defined(MBEDTLS_SSL_CONTEXT_SERIALIZATION) || \ - (defined(MBEDTLS_SSL_KEYING_MATERIAL_EXPORT) && defined(MBEDTLS_SSL_PROTO_TLS1_2)) -#define MBEDTLS_SSL_KEEP_RANDBYTES -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * SSL state machine - */ -typedef enum { - MBEDTLS_SSL_HELLO_REQUEST, - MBEDTLS_SSL_CLIENT_HELLO, - MBEDTLS_SSL_SERVER_HELLO, - MBEDTLS_SSL_SERVER_CERTIFICATE, - MBEDTLS_SSL_SERVER_KEY_EXCHANGE, - MBEDTLS_SSL_CERTIFICATE_REQUEST, - MBEDTLS_SSL_SERVER_HELLO_DONE, - MBEDTLS_SSL_CLIENT_CERTIFICATE, - MBEDTLS_SSL_CLIENT_KEY_EXCHANGE, - MBEDTLS_SSL_CERTIFICATE_VERIFY, - MBEDTLS_SSL_CLIENT_CHANGE_CIPHER_SPEC, - MBEDTLS_SSL_CLIENT_FINISHED, - MBEDTLS_SSL_SERVER_CHANGE_CIPHER_SPEC, - MBEDTLS_SSL_SERVER_FINISHED, - MBEDTLS_SSL_FLUSH_BUFFERS, - MBEDTLS_SSL_HANDSHAKE_WRAPUP, - MBEDTLS_SSL_NEW_SESSION_TICKET, - MBEDTLS_SSL_SERVER_HELLO_VERIFY_REQUEST_SENT, - MBEDTLS_SSL_HELLO_RETRY_REQUEST, - MBEDTLS_SSL_ENCRYPTED_EXTENSIONS, - MBEDTLS_SSL_END_OF_EARLY_DATA, - MBEDTLS_SSL_CLIENT_CERTIFICATE_VERIFY, - MBEDTLS_SSL_CLIENT_CCS_AFTER_SERVER_FINISHED, - MBEDTLS_SSL_CLIENT_CCS_BEFORE_2ND_CLIENT_HELLO, - MBEDTLS_SSL_SERVER_CCS_AFTER_SERVER_HELLO, - MBEDTLS_SSL_CLIENT_CCS_AFTER_CLIENT_HELLO, - MBEDTLS_SSL_SERVER_CCS_AFTER_HELLO_RETRY_REQUEST, - MBEDTLS_SSL_HANDSHAKE_OVER, - MBEDTLS_SSL_TLS1_3_NEW_SESSION_TICKET, - MBEDTLS_SSL_TLS1_3_NEW_SESSION_TICKET_FLUSH, -} -mbedtls_ssl_states; - -/* - * Early data status, client side only. - */ - -#if defined(MBEDTLS_SSL_EARLY_DATA) && defined(MBEDTLS_SSL_CLI_C) -typedef enum { -/* - * See documentation of mbedtls_ssl_get_early_data_status(). - */ - MBEDTLS_SSL_EARLY_DATA_STATUS_NOT_INDICATED, - MBEDTLS_SSL_EARLY_DATA_STATUS_ACCEPTED, - MBEDTLS_SSL_EARLY_DATA_STATUS_REJECTED, -} mbedtls_ssl_early_data_status; -#endif /* MBEDTLS_SSL_EARLY_DATA && MBEDTLS_SSL_CLI_C */ - -/** - * \brief Callback type: send data on the network. - * - * \note That callback may be either blocking or non-blocking. - * - * \param ctx Context for the send callback (typically a file descriptor) - * \param buf Buffer holding the data to send - * \param len Length of the data to send - * - * \return The callback must return the number of bytes sent if any, - * or a non-zero error code. - * If performing non-blocking I/O, \c MBEDTLS_ERR_SSL_WANT_WRITE - * must be returned when the operation would block. - * - * \note The callback is allowed to send fewer bytes than requested. - * It must always return the number of bytes actually sent. - */ -typedef int mbedtls_ssl_send_t(void *ctx, - const unsigned char *buf, - size_t len); - -/** - * \brief Callback type: receive data from the network. - * - * \note That callback may be either blocking or non-blocking. - * - * \param ctx Context for the receive callback (typically a file - * descriptor) - * \param buf Buffer to write the received data to - * \param len Length of the receive buffer - * - * \returns If data has been received, the positive number of bytes received. - * \returns \c 0 if the connection has been closed. - * \returns If performing non-blocking I/O, \c MBEDTLS_ERR_SSL_WANT_READ - * must be returned when the operation would block. - * \returns Another negative error code on other kinds of failures. - * - * \note The callback may receive fewer bytes than the length of the - * buffer. It must always return the number of bytes actually - * received and written to the buffer. - */ -typedef int mbedtls_ssl_recv_t(void *ctx, - unsigned char *buf, - size_t len); - -/** - * \brief Callback type: receive data from the network, with timeout - * - * \note That callback must block until data is received, or the - * timeout delay expires, or the operation is interrupted by a - * signal. - * - * \param ctx Context for the receive callback (typically a file descriptor) - * \param buf Buffer to write the received data to - * \param len Length of the receive buffer - * \param timeout Maximum number of milliseconds to wait for data - * 0 means no timeout (potentially waiting forever) - * - * \return The callback must return the number of bytes received, - * or a non-zero error code: - * \c MBEDTLS_ERR_SSL_TIMEOUT if the operation timed out, - * \c MBEDTLS_ERR_SSL_WANT_READ if interrupted by a signal. - * - * \note The callback may receive fewer bytes than the length of the - * buffer. It must always return the number of bytes actually - * received and written to the buffer. - */ -typedef int mbedtls_ssl_recv_timeout_t(void *ctx, - unsigned char *buf, - size_t len, - uint32_t timeout); -/** - * \brief Callback type: set a pair of timers/delays to watch - * - * \param ctx Context pointer - * \param int_ms Intermediate delay in milliseconds - * \param fin_ms Final delay in milliseconds - * 0 cancels the current timer. - * - * \note This callback must at least store the necessary information - * for the associated \c mbedtls_ssl_get_timer_t callback to - * return correct information. - * - * \note If using an event-driven style of programming, an event must - * be generated when the final delay is passed. The event must - * cause a call to \c mbedtls_ssl_handshake() with the proper - * SSL context to be scheduled. Care must be taken to ensure - * that at most one such call happens at a time. - * - * \note Only one timer at a time must be running. Calling this - * function while a timer is running must cancel it. Cancelled - * timers must not generate any event. - */ -typedef void mbedtls_ssl_set_timer_t(void *ctx, - uint32_t int_ms, - uint32_t fin_ms); - -/** - * \brief Callback type: get status of timers/delays - * - * \param ctx Context pointer - * - * \return This callback must return: - * -1 if cancelled (fin_ms == 0), - * 0 if none of the delays have passed, - * 1 if only the intermediate delay has passed, - * 2 if the final delay has passed. - */ -typedef int mbedtls_ssl_get_timer_t(void *ctx); - -/* Defined below */ -typedef struct mbedtls_ssl_session mbedtls_ssl_session; -typedef struct mbedtls_ssl_context mbedtls_ssl_context; -typedef struct mbedtls_ssl_config mbedtls_ssl_config; - -/* Defined in library/ssl_misc.h */ -typedef struct mbedtls_ssl_transform mbedtls_ssl_transform; -typedef struct mbedtls_ssl_handshake_params mbedtls_ssl_handshake_params; -typedef struct mbedtls_ssl_sig_hash_set_t mbedtls_ssl_sig_hash_set_t; -#if defined(MBEDTLS_X509_CRT_PARSE_C) -typedef struct mbedtls_ssl_key_cert mbedtls_ssl_key_cert; -#endif -#if defined(MBEDTLS_SSL_PROTO_DTLS) -typedef struct mbedtls_ssl_flight_item mbedtls_ssl_flight_item; -#endif - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && defined(MBEDTLS_SSL_SESSION_TICKETS) -#define MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_PSK_RESUMPTION \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK /* 1U << 0 */ -#define MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_PSK_EPHEMERAL_RESUMPTION \ - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL /* 1U << 2 */ -#define MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_EARLY_DATA (1U << 3) - -#define MBEDTLS_SSL_TLS1_3_TICKET_FLAGS_MASK \ - (MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_PSK_RESUMPTION | \ - MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_PSK_EPHEMERAL_RESUMPTION | \ - MBEDTLS_SSL_TLS1_3_TICKET_ALLOW_EARLY_DATA) -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 && MBEDTLS_SSL_SESSION_TICKETS */ - -/** - * \brief Callback type: server-side session cache getter - * - * The session cache is logically a key value store, with - * keys being session IDs and values being instances of - * mbedtls_ssl_session. - * - * This callback retrieves an entry in this key-value store. - * - * \param data The address of the session cache structure to query. - * \param session_id The buffer holding the session ID to query. - * \param session_id_len The length of \p session_id in Bytes. - * \param session The address of the session structure to populate. - * It is initialized with mbdtls_ssl_session_init(), - * and the callback must always leave it in a state - * where it can safely be freed via - * mbedtls_ssl_session_free() independent of the - * return code of this function. - * - * \return \c 0 on success - * \return A non-zero return value on failure. - * - */ -typedef int mbedtls_ssl_cache_get_t(void *data, - unsigned char const *session_id, - size_t session_id_len, - mbedtls_ssl_session *session); -/** - * \brief Callback type: server-side session cache setter - * - * The session cache is logically a key value store, with - * keys being session IDs and values being instances of - * mbedtls_ssl_session. - * - * This callback sets an entry in this key-value store. - * - * \param data The address of the session cache structure to modify. - * \param session_id The buffer holding the session ID to query. - * \param session_id_len The length of \p session_id in Bytes. - * \param session The address of the session to be stored in the - * session cache. - * - * \return \c 0 on success - * \return A non-zero return value on failure. - */ -typedef int mbedtls_ssl_cache_set_t(void *data, - unsigned char const *session_id, - size_t session_id_len, - const mbedtls_ssl_session *session); - -#if defined(MBEDTLS_SSL_ASYNC_PRIVATE) -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Callback type: start external signature operation. - * - * This callback is called during an SSL handshake to start - * a signature decryption operation using an - * external processor. The parameter \p cert contains - * the public key; it is up to the callback function to - * determine how to access the associated private key. - * - * This function typically sends or enqueues a request, and - * does not wait for the operation to complete. This allows - * the handshake step to be non-blocking. - * - * The parameters \p ssl and \p cert are guaranteed to remain - * valid throughout the handshake. On the other hand, this - * function must save the contents of \p hash if the value - * is needed for later processing, because the \p hash buffer - * is no longer valid after this function returns. - * - * This function may call mbedtls_ssl_set_async_operation_data() - * to store an operation context for later retrieval - * by the resume or cancel callback. - * - * \note For RSA signatures, this function must produce output - * that is consistent with PKCS#1 v1.5 in the same way as - * mbedtls_rsa_pkcs1_sign(). Before the private key operation, - * apply the padding steps described in RFC 8017, section 9.2 - * "EMSA-PKCS1-v1_5" as follows. - * - If \p md_alg is #MBEDTLS_MD_NONE, apply the PKCS#1 v1.5 - * encoding, treating \p hash as the DigestInfo to be - * padded. In other words, apply EMSA-PKCS1-v1_5 starting - * from step 3, with `T = hash` and `tLen = hash_len`. - * - If `md_alg != MBEDTLS_MD_NONE`, apply the PKCS#1 v1.5 - * encoding, treating \p hash as the hash to be encoded and - * padded. In other words, apply EMSA-PKCS1-v1_5 starting - * from step 2, with `digestAlgorithm` obtained by calling - * mbedtls_oid_get_oid_by_md() on \p md_alg. - * - * \note For ECDSA signatures, the output format is the DER encoding - * `Ecdsa-Sig-Value` defined in - * [RFC 4492 section 5.4](https://tools.ietf.org/html/rfc4492#section-5.4). - * - * \param ssl The SSL connection instance. It should not be - * modified other than via - * mbedtls_ssl_set_async_operation_data(). - * \param cert Certificate containing the public key. - * In simple cases, this is one of the pointers passed to - * mbedtls_ssl_conf_own_cert() when configuring the SSL - * connection. However, if other callbacks are used, this - * property may not hold. For example, if an SNI callback - * is registered with mbedtls_ssl_conf_sni(), then - * this callback determines what certificate is used. - * \param md_alg Hash algorithm. - * \param hash Buffer containing the hash. This buffer is - * no longer valid when the function returns. - * \param hash_len Size of the \c hash buffer in bytes. - * - * \return 0 if the operation was started successfully and the SSL - * stack should call the resume callback immediately. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if the operation - * was started successfully and the SSL stack should return - * immediately without calling the resume callback yet. - * \return #MBEDTLS_ERR_SSL_HW_ACCEL_FALLTHROUGH if the external - * processor does not support this key. The SSL stack will - * use the private key object instead. - * \return Any other error indicates a fatal failure and is - * propagated up the call chain. The callback should - * use \c MBEDTLS_ERR_PK_xxx error codes, and must not - * use \c MBEDTLS_ERR_SSL_xxx error codes except as - * directed in the documentation of this callback. - */ -typedef int mbedtls_ssl_async_sign_t(mbedtls_ssl_context *ssl, - mbedtls_x509_crt *cert, - mbedtls_md_type_t md_alg, - const unsigned char *hash, - size_t hash_len); - -/** - * \brief Callback type: start external decryption operation. - * - * This callback is called during an SSL handshake to start - * an RSA decryption operation using an - * external processor. The parameter \p cert contains - * the public key; it is up to the callback function to - * determine how to access the associated private key. - * - * This function typically sends or enqueues a request, and - * does not wait for the operation to complete. This allows - * the handshake step to be non-blocking. - * - * The parameters \p ssl and \p cert are guaranteed to remain - * valid throughout the handshake. On the other hand, this - * function must save the contents of \p input if the value - * is needed for later processing, because the \p input buffer - * is no longer valid after this function returns. - * - * This function may call mbedtls_ssl_set_async_operation_data() - * to store an operation context for later retrieval - * by the resume or cancel callback. - * - * \warning RSA decryption as used in TLS is subject to a potential - * timing side channel attack first discovered by Bleichenbacher - * in 1998. This attack can be remotely exploitable - * in practice. To avoid this attack, you must ensure that - * if the callback performs an RSA decryption, the time it - * takes to execute and return the result does not depend - * on whether the RSA decryption succeeded or reported - * invalid padding. - * - * \param ssl The SSL connection instance. It should not be - * modified other than via - * mbedtls_ssl_set_async_operation_data(). - * \param cert Certificate containing the public key. - * In simple cases, this is one of the pointers passed to - * mbedtls_ssl_conf_own_cert() when configuring the SSL - * connection. However, if other callbacks are used, this - * property may not hold. For example, if an SNI callback - * is registered with mbedtls_ssl_conf_sni(), then - * this callback determines what certificate is used. - * \param input Buffer containing the input ciphertext. This buffer - * is no longer valid when the function returns. - * \param input_len Size of the \p input buffer in bytes. - * - * \return 0 if the operation was started successfully and the SSL - * stack should call the resume callback immediately. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if the operation - * was started successfully and the SSL stack should return - * immediately without calling the resume callback yet. - * \return #MBEDTLS_ERR_SSL_HW_ACCEL_FALLTHROUGH if the external - * processor does not support this key. The SSL stack will - * use the private key object instead. - * \return Any other error indicates a fatal failure and is - * propagated up the call chain. The callback should - * use \c MBEDTLS_ERR_PK_xxx error codes, and must not - * use \c MBEDTLS_ERR_SSL_xxx error codes except as - * directed in the documentation of this callback. - */ -typedef int mbedtls_ssl_async_decrypt_t(mbedtls_ssl_context *ssl, - mbedtls_x509_crt *cert, - const unsigned char *input, - size_t input_len); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -/** - * \brief Callback type: resume external operation. - * - * This callback is called during an SSL handshake to resume - * an external operation started by the - * ::mbedtls_ssl_async_sign_t or - * ::mbedtls_ssl_async_decrypt_t callback. - * - * This function typically checks the status of a pending - * request or causes the request queue to make progress, and - * does not wait for the operation to complete. This allows - * the handshake step to be non-blocking. - * - * This function may call mbedtls_ssl_get_async_operation_data() - * to retrieve an operation context set by the start callback. - * It may call mbedtls_ssl_set_async_operation_data() to modify - * this context. - * - * Note that when this function returns a status other than - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS, it must free any - * resources associated with the operation. - * - * \param ssl The SSL connection instance. It should not be - * modified other than via - * mbedtls_ssl_set_async_operation_data(). - * \param output Buffer containing the output (signature or decrypted - * data) on success. - * \param output_len On success, number of bytes written to \p output. - * \param output_size Size of the \p output buffer in bytes. - * - * \return 0 if output of the operation is available in the - * \p output buffer. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if the operation - * is still in progress. Subsequent requests for progress - * on the SSL connection will call the resume callback - * again. - * \return Any other error means that the operation is aborted. - * The SSL handshake is aborted. The callback should - * use \c MBEDTLS_ERR_PK_xxx error codes, and must not - * use \c MBEDTLS_ERR_SSL_xxx error codes except as - * directed in the documentation of this callback. - */ -typedef int mbedtls_ssl_async_resume_t(mbedtls_ssl_context *ssl, - unsigned char *output, - size_t *output_len, - size_t output_size); - -/** - * \brief Callback type: cancel external operation. - * - * This callback is called if an SSL connection is closed - * while an asynchronous operation is in progress. Note that - * this callback is not called if the - * ::mbedtls_ssl_async_resume_t callback has run and has - * returned a value other than - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS, since in that case - * the asynchronous operation has already completed. - * - * This function may call mbedtls_ssl_get_async_operation_data() - * to retrieve an operation context set by the start callback. - * - * \param ssl The SSL connection instance. It should not be - * modified. - */ -typedef void mbedtls_ssl_async_cancel_t(mbedtls_ssl_context *ssl); -#endif /* MBEDTLS_SSL_ASYNC_PRIVATE */ - -#if defined(MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED) && \ - !defined(MBEDTLS_SSL_KEEP_PEER_CERTIFICATE) -#define MBEDTLS_SSL_PEER_CERT_DIGEST_MAX_LEN 48 -#if defined(MBEDTLS_MD_CAN_SHA256) -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_TYPE MBEDTLS_MD_SHA256 -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_LEN 32 -#elif defined(MBEDTLS_MD_CAN_SHA384) -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_TYPE MBEDTLS_MD_SHA384 -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_LEN 48 -#elif defined(MBEDTLS_MD_CAN_SHA1) -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_TYPE MBEDTLS_MD_SHA1 -#define MBEDTLS_SSL_PEER_CERT_DIGEST_DFL_LEN 20 -#else -/* This is already checked in check_config.h, but be sure. */ -#error "Bad configuration - need SHA-1, SHA-256 or SHA-512 enabled to compute digest of peer CRT." -#endif -#endif /* MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED && - !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE */ - -typedef struct { - unsigned char client_application_traffic_secret_N[MBEDTLS_TLS1_3_MD_MAX_SIZE]; - unsigned char server_application_traffic_secret_N[MBEDTLS_TLS1_3_MD_MAX_SIZE]; - unsigned char exporter_master_secret[MBEDTLS_TLS1_3_MD_MAX_SIZE]; - unsigned char resumption_master_secret[MBEDTLS_TLS1_3_MD_MAX_SIZE]; -} mbedtls_ssl_tls13_application_secrets; - -#if defined(MBEDTLS_SSL_DTLS_SRTP) - -#define MBEDTLS_TLS_SRTP_MAX_MKI_LENGTH 255 -#define MBEDTLS_TLS_SRTP_MAX_PROFILE_LIST_LENGTH 4 -/* - * For code readability use a typedef for DTLS-SRTP profiles - * - * Use_srtp extension protection profiles values as defined in - * http://www.iana.org/assignments/srtp-protection/srtp-protection.xhtml - * - * Reminder: if this list is expanded mbedtls_ssl_check_srtp_profile_value - * must be updated too. - */ -#define MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_80 ((uint16_t) 0x0001) -#define MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_32 ((uint16_t) 0x0002) -#define MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_80 ((uint16_t) 0x0005) -#define MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_32 ((uint16_t) 0x0006) -/* This one is not iana defined, but for code readability. */ -#define MBEDTLS_TLS_SRTP_UNSET ((uint16_t) 0x0000) - -typedef uint16_t mbedtls_ssl_srtp_profile; - -typedef struct mbedtls_dtls_srtp_info_t { - /*! The SRTP profile that was negotiated. */ - mbedtls_ssl_srtp_profile MBEDTLS_PRIVATE(chosen_dtls_srtp_profile); - /*! The length of mki_value. */ - uint16_t MBEDTLS_PRIVATE(mki_len); - /*! The mki_value used, with max size of 256 bytes. */ - unsigned char MBEDTLS_PRIVATE(mki_value)[MBEDTLS_TLS_SRTP_MAX_MKI_LENGTH]; -} -mbedtls_dtls_srtp_info; - -#endif /* MBEDTLS_SSL_DTLS_SRTP */ - -/** Human-friendly representation of the (D)TLS protocol version. */ -typedef enum { - MBEDTLS_SSL_VERSION_UNKNOWN, /*!< Context not in use or version not yet negotiated. */ - MBEDTLS_SSL_VERSION_TLS1_2 = 0x0303, /*!< (D)TLS 1.2 */ - MBEDTLS_SSL_VERSION_TLS1_3 = 0x0304, /*!< (D)TLS 1.3 */ -} mbedtls_ssl_protocol_version; - -/* - * This structure is used for storing current session data. - * - * Note: when changing this definition, we need to check and update: - * - in tests/suites/test_suite_ssl.function: - * ssl_populate_session() and ssl_serialize_session_save_load() - * - in library/ssl_tls.c: - * mbedtls_ssl_session_init() and mbedtls_ssl_session_free() - * mbedtls_ssl_session_save() and ssl_session_load() - * ssl_session_copy() - */ -struct mbedtls_ssl_session { -#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH) - unsigned char MBEDTLS_PRIVATE(mfl_code); /*!< MaxFragmentLength negotiated by peer */ -#endif /* MBEDTLS_SSL_MAX_FRAGMENT_LENGTH */ - -/*!< RecordSizeLimit received from the peer */ -#if defined(MBEDTLS_SSL_RECORD_SIZE_LIMIT) - uint16_t MBEDTLS_PRIVATE(record_size_limit); -#endif /* MBEDTLS_SSL_RECORD_SIZE_LIMIT */ - - unsigned char MBEDTLS_PRIVATE(exported); - uint8_t MBEDTLS_PRIVATE(endpoint); /*!< 0: client, 1: server */ - - /** TLS version negotiated in the session. Used if and when renegotiating - * or resuming a session instead of the configured minor TLS version. - */ - mbedtls_ssl_protocol_version MBEDTLS_PRIVATE(tls_version); - -#if defined(MBEDTLS_HAVE_TIME) - mbedtls_time_t MBEDTLS_PRIVATE(start); /*!< start time of current session */ -#endif - int MBEDTLS_PRIVATE(ciphersuite); /*!< chosen ciphersuite */ - size_t MBEDTLS_PRIVATE(id_len); /*!< session id length */ - unsigned char MBEDTLS_PRIVATE(id)[32]; /*!< session identifier */ - unsigned char MBEDTLS_PRIVATE(master)[48]; /*!< the master secret */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -#if defined(MBEDTLS_SSL_KEEP_PEER_CERTIFICATE) - mbedtls_x509_crt *MBEDTLS_PRIVATE(peer_cert); /*!< peer X.509 cert chain */ -#else /* MBEDTLS_SSL_KEEP_PEER_CERTIFICATE */ - /*! The digest of the peer's end-CRT. This must be kept to detect CRT - * changes during renegotiation, mitigating the triple handshake attack. */ - unsigned char *MBEDTLS_PRIVATE(peer_cert_digest); - size_t MBEDTLS_PRIVATE(peer_cert_digest_len); - mbedtls_md_type_t MBEDTLS_PRIVATE(peer_cert_digest_type); -#endif /* !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE */ -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - uint32_t MBEDTLS_PRIVATE(verify_result); /*!< verification result */ - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && defined(MBEDTLS_SSL_CLI_C) - unsigned char *MBEDTLS_PRIVATE(ticket); /*!< RFC 5077 session ticket */ - size_t MBEDTLS_PRIVATE(ticket_len); /*!< session ticket length */ - uint32_t MBEDTLS_PRIVATE(ticket_lifetime); /*!< ticket lifetime hint */ -#endif /* MBEDTLS_SSL_SESSION_TICKETS && MBEDTLS_SSL_CLI_C */ - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && defined(MBEDTLS_SSL_SRV_C) && \ - defined(MBEDTLS_HAVE_TIME) - /*! When a ticket is created by a TLS server as part of an established TLS - * session, the ticket creation time may need to be saved for the ticket - * module to be able to check the ticket age when the ticket is used. - * That's the purpose of this field. - * Before creating a new ticket, an Mbed TLS server set this field with - * its current time in milliseconds. This time may then be saved in the - * session ticket data by the session ticket writing function and - * recovered by the ticket parsing function later when the ticket is used. - * The ticket module may then use this time to compute the ticket age and - * determine if it has expired or not. - * The Mbed TLS implementations of the session ticket writing and parsing - * functions save and retrieve the ticket creation time as part of the - * session ticket data. The session ticket parsing function relies on - * the mbedtls_ssl_session_get_ticket_creation_time() API to get the - * ticket creation time from the session ticket data. - */ - mbedtls_ms_time_t MBEDTLS_PRIVATE(ticket_creation_time); -#endif - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) && defined(MBEDTLS_SSL_SESSION_TICKETS) - uint32_t MBEDTLS_PRIVATE(ticket_age_add); /*!< Randomly generated value used to obscure the age of the ticket */ - uint8_t MBEDTLS_PRIVATE(ticket_flags); /*!< Ticket flags */ - uint8_t MBEDTLS_PRIVATE(resumption_key_len); /*!< resumption_key length */ - unsigned char MBEDTLS_PRIVATE(resumption_key)[MBEDTLS_SSL_TLS1_3_TICKET_RESUMPTION_KEY_LEN]; - -#if defined(MBEDTLS_SSL_SERVER_NAME_INDICATION) && defined(MBEDTLS_SSL_CLI_C) - char *MBEDTLS_PRIVATE(hostname); /*!< host name binded with tickets */ -#endif /* MBEDTLS_SSL_SERVER_NAME_INDICATION && MBEDTLS_SSL_CLI_C */ - -#if defined(MBEDTLS_SSL_EARLY_DATA) && defined(MBEDTLS_SSL_ALPN) && defined(MBEDTLS_SSL_SRV_C) - char *ticket_alpn; /*!< ALPN negotiated in the session - during which the ticket was generated. */ -#endif - -#if defined(MBEDTLS_HAVE_TIME) && defined(MBEDTLS_SSL_CLI_C) - /*! Time in milliseconds when the last ticket was received. */ - mbedtls_ms_time_t MBEDTLS_PRIVATE(ticket_reception_time); -#endif -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 && MBEDTLS_SSL_SESSION_TICKETS */ - -#if defined(MBEDTLS_SSL_EARLY_DATA) - uint32_t MBEDTLS_PRIVATE(max_early_data_size); /*!< maximum amount of early data in tickets */ -#endif - -#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC) - int MBEDTLS_PRIVATE(encrypt_then_mac); /*!< flag for EtM activation */ -#endif - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) - mbedtls_ssl_tls13_application_secrets MBEDTLS_PRIVATE(app_secrets); -#endif -}; - -/* - * Identifiers for PRFs used in various versions of TLS. - */ -typedef enum { - MBEDTLS_SSL_TLS_PRF_NONE, - MBEDTLS_SSL_TLS_PRF_SHA384, - MBEDTLS_SSL_TLS_PRF_SHA256, - MBEDTLS_SSL_HKDF_EXPAND_SHA384, - MBEDTLS_SSL_HKDF_EXPAND_SHA256 -} -mbedtls_tls_prf_types; - -typedef enum { - MBEDTLS_SSL_KEY_EXPORT_TLS12_MASTER_SECRET = 0, -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_EARLY_SECRET, - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_EARLY_EXPORTER_SECRET, - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_HANDSHAKE_TRAFFIC_SECRET, - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_SERVER_HANDSHAKE_TRAFFIC_SECRET, - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_CLIENT_APPLICATION_TRAFFIC_SECRET, - MBEDTLS_SSL_KEY_EXPORT_TLS1_3_SERVER_APPLICATION_TRAFFIC_SECRET, -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 */ -} mbedtls_ssl_key_export_type; - -/** - * \brief Callback type: Export key alongside random values for - * session identification, and PRF for - * implementation of TLS key exporters. - * - * \param p_expkey Context for the callback. - * \param type The type of the key that is being exported. - * \param secret The address of the buffer holding the secret - * that's being exporterd. - * \param secret_len The length of \p secret in bytes. - * \param client_random The client random bytes. - * \param server_random The server random bytes. - * \param tls_prf_type The identifier for the PRF used in the handshake - * to which the key belongs. - */ -typedef void mbedtls_ssl_export_keys_t(void *p_expkey, - mbedtls_ssl_key_export_type type, - const unsigned char *secret, - size_t secret_len, - const unsigned char client_random[32], - const unsigned char server_random[32], - mbedtls_tls_prf_types tls_prf_type); - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Callback type: generic handshake callback - * - * \note Callbacks may use user_data funcs to set/get app user data. - * See \c mbedtls_ssl_get_user_data_p() - * \c mbedtls_ssl_get_user_data_n() - * \c mbedtls_ssl_conf_get_user_data_p() - * \c mbedtls_ssl_conf_get_user_data_n() - * - * \param ssl \c mbedtls_ssl_context on which the callback is run - * - * \return The return value of the callback is 0 if successful, - * or a specific MBEDTLS_ERR_XXX code, which will cause - * the handshake to be aborted. - */ -typedef int (*mbedtls_ssl_hs_cb_t)(mbedtls_ssl_context *ssl); -#endif - -/* A type for storing user data in a library structure. - * - * The representation of type may change in future versions of the library. - * Only the behaviors guaranteed by documented accessor functions are - * guaranteed to remain stable. - */ -typedef union { - uintptr_t n; /* typically a handle to an associated object */ - void *p; /* typically a pointer to extra data */ -} mbedtls_ssl_user_data_t; - -/** - * SSL/TLS configuration to be shared between mbedtls_ssl_context structures. - */ -struct mbedtls_ssl_config { - /* Group items mostly by size. This helps to reduce memory wasted to - * padding. It also helps to keep smaller fields early in the structure, - * so that elements tend to be in the 128-element direct access window - * on Arm Thumb, which reduces the code size. */ - - mbedtls_ssl_protocol_version MBEDTLS_PRIVATE(max_tls_version); /*!< max. TLS version used */ - mbedtls_ssl_protocol_version MBEDTLS_PRIVATE(min_tls_version); /*!< min. TLS version used */ - - /* - * Flags (could be bit-fields to save RAM, but separate bytes make - * the code smaller on architectures with an instruction for direct - * byte access). - */ - - uint8_t MBEDTLS_PRIVATE(endpoint); /*!< 0: client, 1: server */ - uint8_t MBEDTLS_PRIVATE(transport); /*!< 0: stream (TLS), 1: datagram (DTLS) */ - uint8_t MBEDTLS_PRIVATE(authmode); /*!< MBEDTLS_SSL_VERIFY_XXX */ - /* needed even with renego disabled for LEGACY_BREAK_HANDSHAKE */ - uint8_t MBEDTLS_PRIVATE(allow_legacy_renegotiation); /*!< MBEDTLS_LEGACY_XXX */ -#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH) - uint8_t MBEDTLS_PRIVATE(mfl_code); /*!< desired fragment length indicator - (MBEDTLS_SSL_MAX_FRAG_LEN_XXX) */ -#endif -#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC) - uint8_t MBEDTLS_PRIVATE(encrypt_then_mac); /*!< negotiate encrypt-then-mac? */ -#endif -#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET) - uint8_t MBEDTLS_PRIVATE(extended_ms); /*!< negotiate extended master secret? */ -#endif -#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY) - uint8_t MBEDTLS_PRIVATE(anti_replay); /*!< detect and prevent replay? */ -#endif -#if defined(MBEDTLS_SSL_RENEGOTIATION) - uint8_t MBEDTLS_PRIVATE(disable_renegotiation); /*!< disable renegotiation? */ -#endif -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && \ - defined(MBEDTLS_SSL_CLI_C) - /** Encodes two booleans, one stating whether TLS 1.2 session tickets are - * enabled or not, the other one whether the handling of TLS 1.3 - * NewSessionTicket messages is enabled or not. They are respectively set - * by mbedtls_ssl_conf_session_tickets() and - * mbedtls_ssl_conf_tls13_enable_signal_new_session_tickets(). - */ - uint8_t MBEDTLS_PRIVATE(session_tickets); /*!< use session tickets? */ -#endif - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && \ - defined(MBEDTLS_SSL_SRV_C) && \ - defined(MBEDTLS_SSL_PROTO_TLS1_3) - uint16_t MBEDTLS_PRIVATE(new_session_tickets_count); /*!< number of NewSessionTicket */ -#endif - -#if defined(MBEDTLS_SSL_SRV_C) - uint8_t MBEDTLS_PRIVATE(cert_req_ca_list); /*!< enable sending CA list in - Certificate Request messages? */ - uint8_t MBEDTLS_PRIVATE(respect_cli_pref); /*!< pick the ciphersuite according to - the client's preferences rather - than ours? */ -#endif -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - uint8_t MBEDTLS_PRIVATE(ignore_unexpected_cid); /*!< Should DTLS record with - * unexpected CID - * lead to failure? */ -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ -#if defined(MBEDTLS_SSL_DTLS_SRTP) - uint8_t MBEDTLS_PRIVATE(dtls_srtp_mki_support); /* support having mki_value - in the use_srtp extension? */ -#endif - - /* - * Pointers - */ - - /** Allowed ciphersuites for (D)TLS 1.2 (0-terminated) */ - const int *MBEDTLS_PRIVATE(ciphersuite_list); - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) - /** Allowed TLS 1.3 key exchange modes. */ - int MBEDTLS_PRIVATE(tls13_kex_modes); -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 */ - - /** Callback for printing debug output */ - void(*MBEDTLS_PRIVATE(f_dbg))(void *, int, const char *, int, const char *); - void *MBEDTLS_PRIVATE(p_dbg); /*!< context for the debug function */ - - /** Callback for getting (pseudo-)random numbers */ - int(*MBEDTLS_PRIVATE(f_rng))(void *, unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_rng); /*!< context for the RNG function */ - - /** Callback to retrieve a session from the cache */ - mbedtls_ssl_cache_get_t *MBEDTLS_PRIVATE(f_get_cache); - /** Callback to store a session into the cache */ - mbedtls_ssl_cache_set_t *MBEDTLS_PRIVATE(f_set_cache); - void *MBEDTLS_PRIVATE(p_cache); /*!< context for cache callbacks */ - -#if defined(MBEDTLS_SSL_SERVER_NAME_INDICATION) - /** Callback for setting cert according to SNI extension */ - int(*MBEDTLS_PRIVATE(f_sni))(void *, mbedtls_ssl_context *, const unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_sni); /*!< context for SNI callback */ -#endif - -#if defined(MBEDTLS_X509_CRT_PARSE_C) - /** Callback to customize X.509 certificate chain verification */ - int(*MBEDTLS_PRIVATE(f_vrfy))(void *, mbedtls_x509_crt *, int, uint32_t *); - void *MBEDTLS_PRIVATE(p_vrfy); /*!< context for X.509 verify calllback */ -#endif - -#if defined(MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED) -#if defined(MBEDTLS_SSL_SRV_C) - /** Callback to retrieve PSK key from identity */ - int(*MBEDTLS_PRIVATE(f_psk))(void *, mbedtls_ssl_context *, const unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_psk); /*!< context for PSK callback */ -#endif -#endif - -#if defined(MBEDTLS_SSL_DTLS_HELLO_VERIFY) && defined(MBEDTLS_SSL_SRV_C) - /** Callback to create & write a cookie for ClientHello verification */ - int(*MBEDTLS_PRIVATE(f_cookie_write))(void *, unsigned char **, unsigned char *, - const unsigned char *, size_t); - /** Callback to verify validity of a ClientHello cookie */ - int(*MBEDTLS_PRIVATE(f_cookie_check))(void *, const unsigned char *, size_t, - const unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_cookie); /*!< context for the cookie callbacks */ -#endif - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && defined(MBEDTLS_SSL_SRV_C) - /** Callback to create & write a session ticket */ - int(*MBEDTLS_PRIVATE(f_ticket_write))(void *, const mbedtls_ssl_session *, - unsigned char *, const unsigned char *, size_t *, - uint32_t *); - /** Callback to parse a session ticket into a session structure */ - int(*MBEDTLS_PRIVATE(f_ticket_parse))(void *, mbedtls_ssl_session *, unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_ticket); /*!< context for the ticket callbacks */ -#endif /* MBEDTLS_SSL_SESSION_TICKETS && MBEDTLS_SSL_SRV_C */ -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - size_t MBEDTLS_PRIVATE(cid_len); /*!< The length of CIDs for incoming DTLS records. */ -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) - const mbedtls_x509_crt_profile *MBEDTLS_PRIVATE(cert_profile); /*!< verification profile */ - mbedtls_ssl_key_cert *MBEDTLS_PRIVATE(key_cert); /*!< own certificate/key pair(s) */ - mbedtls_x509_crt *MBEDTLS_PRIVATE(ca_chain); /*!< trusted CAs */ - mbedtls_x509_crl *MBEDTLS_PRIVATE(ca_crl); /*!< trusted CAs CRLs */ -#if defined(MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK) - mbedtls_x509_crt_ca_cb_t MBEDTLS_PRIVATE(f_ca_cb); - void *MBEDTLS_PRIVATE(p_ca_cb); -#endif /* MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK */ -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_SSL_ASYNC_PRIVATE) -#if defined(MBEDTLS_X509_CRT_PARSE_C) - mbedtls_ssl_async_sign_t *MBEDTLS_PRIVATE(f_async_sign_start); /*!< start asynchronous signature operation */ - mbedtls_ssl_async_decrypt_t *MBEDTLS_PRIVATE(f_async_decrypt_start); /*!< start asynchronous decryption operation */ -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - mbedtls_ssl_async_resume_t *MBEDTLS_PRIVATE(f_async_resume); /*!< resume asynchronous operation */ - mbedtls_ssl_async_cancel_t *MBEDTLS_PRIVATE(f_async_cancel); /*!< cancel asynchronous operation */ - void *MBEDTLS_PRIVATE(p_async_config_data); /*!< Configuration data set by mbedtls_ssl_conf_async_private_cb(). */ -#endif /* MBEDTLS_SSL_ASYNC_PRIVATE */ - -#if defined(MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED) - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) - const int *MBEDTLS_PRIVATE(sig_hashes); /*!< allowed signature hashes */ -#endif - const uint16_t *MBEDTLS_PRIVATE(sig_algs); /*!< allowed signature algorithms */ -#endif /* MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED */ - -#if defined(MBEDTLS_ECP_C) && !defined(MBEDTLS_DEPRECATED_REMOVED) - const mbedtls_ecp_group_id *MBEDTLS_PRIVATE(curve_list); /*!< allowed curves */ -#endif - - const uint16_t *MBEDTLS_PRIVATE(group_list); /*!< allowed IANA NamedGroups */ - -#if defined(MBEDTLS_DHM_C) - mbedtls_mpi MBEDTLS_PRIVATE(dhm_P); /*!< prime modulus for DHM */ - mbedtls_mpi MBEDTLS_PRIVATE(dhm_G); /*!< generator for DHM */ -#endif - -#if defined(MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED) - -#if defined(MBEDTLS_USE_PSA_CRYPTO) - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(psk_opaque); /*!< PSA key slot holding opaque PSK. This field - * should only be set via - * mbedtls_ssl_conf_psk_opaque(). - * If either no PSK or a raw PSK have been - * configured, this has value \c 0. - */ -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - unsigned char *MBEDTLS_PRIVATE(psk); /*!< The raw pre-shared key. This field should - * only be set via mbedtls_ssl_conf_psk(). - * If either no PSK or an opaque PSK - * have been configured, this has value NULL. */ - size_t MBEDTLS_PRIVATE(psk_len); /*!< The length of the raw pre-shared key. - * This field should only be set via - * mbedtls_ssl_conf_psk(). - * Its value is non-zero if and only if - * \c psk is not \c NULL. */ - - unsigned char *MBEDTLS_PRIVATE(psk_identity); /*!< The PSK identity for PSK negotiation. - * This field should only be set via - * mbedtls_ssl_conf_psk(). - * This is set if and only if either - * \c psk or \c psk_opaque are set. */ - size_t MBEDTLS_PRIVATE(psk_identity_len);/*!< The length of PSK identity. - * This field should only be set via - * mbedtls_ssl_conf_psk(). - * Its value is non-zero if and only if - * \c psk is not \c NULL or \c psk_opaque - * is not \c 0. */ -#endif /* MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED */ - -#if defined(MBEDTLS_SSL_EARLY_DATA) - int MBEDTLS_PRIVATE(early_data_enabled); /*!< Early data enablement: - * - MBEDTLS_SSL_EARLY_DATA_DISABLED, - * - MBEDTLS_SSL_EARLY_DATA_ENABLED */ - -#if defined(MBEDTLS_SSL_SRV_C) - /* The maximum amount of 0-RTT data. RFC 8446 section 4.6.1 */ - uint32_t MBEDTLS_PRIVATE(max_early_data_size); -#endif /* MBEDTLS_SSL_SRV_C */ - -#endif /* MBEDTLS_SSL_EARLY_DATA */ - -#if defined(MBEDTLS_SSL_ALPN) - const char **MBEDTLS_PRIVATE(alpn_list); /*!< ordered list of protocols */ -#endif - -#if defined(MBEDTLS_SSL_DTLS_SRTP) - /*! ordered list of supported srtp profile */ - const mbedtls_ssl_srtp_profile *MBEDTLS_PRIVATE(dtls_srtp_profile_list); - /*! number of supported profiles */ - size_t MBEDTLS_PRIVATE(dtls_srtp_profile_list_len); -#endif /* MBEDTLS_SSL_DTLS_SRTP */ - - /* - * Numerical settings (int) - */ - - uint32_t MBEDTLS_PRIVATE(read_timeout); /*!< timeout for mbedtls_ssl_read (ms) */ - -#if defined(MBEDTLS_SSL_PROTO_DTLS) - uint32_t MBEDTLS_PRIVATE(hs_timeout_min); /*!< initial value of the handshake - retransmission timeout (ms) */ - uint32_t MBEDTLS_PRIVATE(hs_timeout_max); /*!< maximum value of the handshake - retransmission timeout (ms) */ -#endif - -#if defined(MBEDTLS_SSL_RENEGOTIATION) - int MBEDTLS_PRIVATE(renego_max_records); /*!< grace period for renegotiation */ - unsigned char MBEDTLS_PRIVATE(renego_period)[8]; /*!< value of the record counters - that triggers renegotiation */ -#endif - - unsigned int MBEDTLS_PRIVATE(badmac_limit); /*!< limit of records with a bad MAC */ - -#if defined(MBEDTLS_DHM_C) && defined(MBEDTLS_SSL_CLI_C) - unsigned int MBEDTLS_PRIVATE(dhm_min_bitlen); /*!< min. bit length of the DHM prime */ -#endif - - /** User data pointer or handle. - * - * The library sets this to \p 0 when creating a context and does not - * access it afterwards. - */ - mbedtls_ssl_user_data_t MBEDTLS_PRIVATE(user_data); - -#if defined(MBEDTLS_SSL_SRV_C) - mbedtls_ssl_hs_cb_t MBEDTLS_PRIVATE(f_cert_cb); /*!< certificate selection callback */ -#endif /* MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED) - const mbedtls_x509_crt *MBEDTLS_PRIVATE(dn_hints);/*!< acceptable client cert issuers */ -#endif -}; - -struct mbedtls_ssl_context { - const mbedtls_ssl_config *MBEDTLS_PRIVATE(conf); /*!< configuration information */ - - /* - * Miscellaneous - */ - int MBEDTLS_PRIVATE(state); /*!< SSL handshake: current state */ -#if defined(MBEDTLS_SSL_RENEGOTIATION) - int MBEDTLS_PRIVATE(renego_status); /*!< Initial, in progress, pending? */ - int MBEDTLS_PRIVATE(renego_records_seen); /*!< Records since renego request, or with DTLS, - number of retransmissions of request if - renego_max_records is < 0 */ -#endif /* MBEDTLS_SSL_RENEGOTIATION */ - - /** - * Maximum TLS version to be negotiated, then negotiated TLS version. - * - * It is initialized as the configured maximum TLS version to be - * negotiated by mbedtls_ssl_setup(). - * - * When renegotiating or resuming a session, it is overwritten in the - * ClientHello writing preparation stage with the previously negotiated - * TLS version. - * - * On client side, it is updated to the TLS version selected by the server - * for the handshake when the ServerHello is received. - * - * On server side, it is updated to the TLS version the server selects for - * the handshake when the ClientHello is received. - */ - mbedtls_ssl_protocol_version MBEDTLS_PRIVATE(tls_version); - -#if defined(MBEDTLS_SSL_EARLY_DATA) && defined(MBEDTLS_SSL_CLI_C) - /** - * State of the negotiation and transfer of early data. Reset to - * MBEDTLS_SSL_EARLY_DATA_STATE_IDLE when the context is reset. - */ - int MBEDTLS_PRIVATE(early_data_state); -#endif - - /** Multipurpose field. - * - * - DTLS: records with a bad MAC received. - * - TLS: accumulated length of handshake fragments (up to \c in_hslen). - * - * This field is multipurpose in order to preserve the ABI in the - * Mbed TLS 3.6 LTS branch. Until 3.6.2, it was only used in DTLS - * and called `badmac_seen`. - */ - unsigned MBEDTLS_PRIVATE(badmac_seen_or_in_hsfraglen); - -#if defined(MBEDTLS_X509_CRT_PARSE_C) - /** Callback to customize X.509 certificate chain verification */ - int(*MBEDTLS_PRIVATE(f_vrfy))(void *, mbedtls_x509_crt *, int, uint32_t *); - void *MBEDTLS_PRIVATE(p_vrfy); /*!< context for X.509 verify callback */ -#endif - - mbedtls_ssl_send_t *MBEDTLS_PRIVATE(f_send); /*!< Callback for network send */ - mbedtls_ssl_recv_t *MBEDTLS_PRIVATE(f_recv); /*!< Callback for network receive */ - mbedtls_ssl_recv_timeout_t *MBEDTLS_PRIVATE(f_recv_timeout); - /*!< Callback for network receive with timeout */ - - void *MBEDTLS_PRIVATE(p_bio); /*!< context for I/O operations */ - - /* - * Session layer - */ - mbedtls_ssl_session *MBEDTLS_PRIVATE(session_in); /*!< current session data (in) */ - mbedtls_ssl_session *MBEDTLS_PRIVATE(session_out); /*!< current session data (out) */ - mbedtls_ssl_session *MBEDTLS_PRIVATE(session); /*!< negotiated session data */ - mbedtls_ssl_session *MBEDTLS_PRIVATE(session_negotiate); /*!< session data in negotiation */ - - mbedtls_ssl_handshake_params *MBEDTLS_PRIVATE(handshake); /*!< params required only during - the handshake process */ - - /* - * Record layer transformations - */ - mbedtls_ssl_transform *MBEDTLS_PRIVATE(transform_in); /*!< current transform params (in) - * This is always a reference, - * never an owning pointer. */ - mbedtls_ssl_transform *MBEDTLS_PRIVATE(transform_out); /*!< current transform params (out) - * This is always a reference, - * never an owning pointer. */ - mbedtls_ssl_transform *MBEDTLS_PRIVATE(transform); /*!< negotiated transform params - * This pointer owns the transform - * it references. */ -#if defined(MBEDTLS_SSL_PROTO_TLS1_2) - mbedtls_ssl_transform *MBEDTLS_PRIVATE(transform_negotiate); /*!< transform params in negotiation - * This pointer owns the transform - * it references. */ -#endif /* MBEDTLS_SSL_PROTO_TLS1_2 */ - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) - /*! The application data transform in TLS 1.3. - * This pointer owns the transform it references. */ - mbedtls_ssl_transform *MBEDTLS_PRIVATE(transform_application); -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 */ - - /* - * Timers - */ - void *MBEDTLS_PRIVATE(p_timer); /*!< context for the timer callbacks */ - - mbedtls_ssl_set_timer_t *MBEDTLS_PRIVATE(f_set_timer); /*!< set timer callback */ - mbedtls_ssl_get_timer_t *MBEDTLS_PRIVATE(f_get_timer); /*!< get timer callback */ - - /* - * Record layer (incoming data) - */ - unsigned char *MBEDTLS_PRIVATE(in_buf); /*!< input buffer */ - unsigned char *MBEDTLS_PRIVATE(in_ctr); /*!< 64-bit incoming message counter - TLS: maintained by us - DTLS: read from peer */ - unsigned char *MBEDTLS_PRIVATE(in_hdr); /*!< start of record header */ -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - unsigned char *MBEDTLS_PRIVATE(in_cid); /*!< The start of the CID; - * (the end is marked by in_len). */ -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - unsigned char *MBEDTLS_PRIVATE(in_len); /*!< two-bytes message length field */ - unsigned char *MBEDTLS_PRIVATE(in_iv); /*!< ivlen-byte IV */ - unsigned char *MBEDTLS_PRIVATE(in_msg); /*!< message contents (in_iv+ivlen) */ - unsigned char *MBEDTLS_PRIVATE(in_offt); /*!< read offset in application data */ - - int MBEDTLS_PRIVATE(in_msgtype); /*!< record header: message type */ - size_t MBEDTLS_PRIVATE(in_msglen); /*!< record header: message length */ - size_t MBEDTLS_PRIVATE(in_left); /*!< amount of data read so far */ -#if defined(MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH) - size_t MBEDTLS_PRIVATE(in_buf_len); /*!< length of input buffer */ -#endif -#if defined(MBEDTLS_SSL_PROTO_DTLS) - uint16_t MBEDTLS_PRIVATE(in_epoch); /*!< DTLS epoch for incoming records */ - size_t MBEDTLS_PRIVATE(next_record_offset); /*!< offset of the next record in datagram - (equal to in_left if none) */ -#endif /* MBEDTLS_SSL_PROTO_DTLS */ -#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY) - uint64_t MBEDTLS_PRIVATE(in_window_top); /*!< last validated record seq_num */ - uint64_t MBEDTLS_PRIVATE(in_window); /*!< bitmask for replay detection */ -#endif /* MBEDTLS_SSL_DTLS_ANTI_REPLAY */ - - size_t MBEDTLS_PRIVATE(in_hslen); /*!< current handshake message length, - including the handshake header */ - int MBEDTLS_PRIVATE(nb_zero); /*!< # of 0-length encrypted messages */ - - int MBEDTLS_PRIVATE(keep_current_message); /*!< drop or reuse current message - on next call to record layer? */ - - /* The following three variables indicate if and, if yes, - * what kind of alert is pending to be sent. - */ - unsigned char MBEDTLS_PRIVATE(send_alert); /*!< Determines if a fatal alert - should be sent. Values: - - \c 0 , no alert is to be sent. - - \c 1 , alert is to be sent. */ - unsigned char MBEDTLS_PRIVATE(alert_type); /*!< Type of alert if send_alert - != 0 */ - int MBEDTLS_PRIVATE(alert_reason); /*!< The error code to be returned - to the user once the fatal alert - has been sent. */ - -#if defined(MBEDTLS_SSL_PROTO_DTLS) - uint8_t MBEDTLS_PRIVATE(disable_datagram_packing); /*!< Disable packing multiple records - * within a single datagram. */ -#endif /* MBEDTLS_SSL_PROTO_DTLS */ - -#if defined(MBEDTLS_SSL_EARLY_DATA) -#if defined(MBEDTLS_SSL_SRV_C) - /* - * One of: - * MBEDTLS_SSL_EARLY_DATA_NO_DISCARD - * MBEDTLS_SSL_EARLY_DATA_TRY_TO_DEPROTECT_AND_DISCARD - * MBEDTLS_SSL_EARLY_DATA_DISCARD - */ - uint8_t MBEDTLS_PRIVATE(discard_early_data_record); -#endif - uint32_t MBEDTLS_PRIVATE(total_early_data_size); /*!< Number of received/written early data bytes */ -#endif /* MBEDTLS_SSL_EARLY_DATA */ - - /* - * Record layer (outgoing data) - */ - unsigned char *MBEDTLS_PRIVATE(out_buf); /*!< output buffer */ - unsigned char *MBEDTLS_PRIVATE(out_ctr); /*!< 64-bit outgoing message counter */ - unsigned char *MBEDTLS_PRIVATE(out_hdr); /*!< start of record header */ -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - unsigned char *MBEDTLS_PRIVATE(out_cid); /*!< The start of the CID; - * (the end is marked by in_len). */ -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - unsigned char *MBEDTLS_PRIVATE(out_len); /*!< two-bytes message length field */ - unsigned char *MBEDTLS_PRIVATE(out_iv); /*!< ivlen-byte IV */ - unsigned char *MBEDTLS_PRIVATE(out_msg); /*!< message contents (out_iv+ivlen) */ - - int MBEDTLS_PRIVATE(out_msgtype); /*!< record header: message type */ - size_t MBEDTLS_PRIVATE(out_msglen); /*!< record header: message length */ - size_t MBEDTLS_PRIVATE(out_left); /*!< amount of data not yet written */ -#if defined(MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH) - size_t MBEDTLS_PRIVATE(out_buf_len); /*!< length of output buffer */ -#endif - - unsigned char MBEDTLS_PRIVATE(cur_out_ctr)[MBEDTLS_SSL_SEQUENCE_NUMBER_LEN]; /*!< Outgoing record sequence number. */ - -#if defined(MBEDTLS_SSL_PROTO_DTLS) - uint16_t MBEDTLS_PRIVATE(mtu); /*!< path mtu, used to fragment outgoing messages */ -#endif /* MBEDTLS_SSL_PROTO_DTLS */ - - /* - * User settings - */ -#if defined(MBEDTLS_X509_CRT_PARSE_C) - /** Expected peer CN for verification. - * - * Also used on clients for SNI, - * and for TLS 1.3 session resumption using tickets. - * - * The value of this field can be: - * - \p NULL in a newly initialized or reset context. - * - A heap-allocated copy of the last value passed to - * mbedtls_ssl_set_hostname(), if the last call had a non-null - * \p hostname argument. - * - A special value to indicate that mbedtls_ssl_set_hostname() - * was called with \p NULL (as opposed to never having been called). - * See `mbedtls_ssl_get_hostname_pointer()` in `ssl_tls.c`. - * - * If this field contains the value \p NULL and the configuration option - * #MBEDTLS_SSL_CLI_ALLOW_WEAK_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * is unset, on a TLS client, attempting to verify a server certificate - * results in the error - * #MBEDTLS_ERR_SSL_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME. - * - * If this field contains the special value described above, or if - * the value is \p NULL and the configuration option - * #MBEDTLS_SSL_CLI_ALLOW_WEAK_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * is set, then the peer name verification is skipped, which may be - * insecure, especially on a client. Furthermore, on a client, the - * server_name extension is not sent, and the server name is ignored - * in TLS 1.3 session resumption using tickets. - */ - char *MBEDTLS_PRIVATE(hostname); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_SSL_ALPN) - const char *MBEDTLS_PRIVATE(alpn_chosen); /*!< negotiated protocol */ -#endif /* MBEDTLS_SSL_ALPN */ - -#if defined(MBEDTLS_SSL_DTLS_SRTP) - /* - * use_srtp extension - */ - mbedtls_dtls_srtp_info MBEDTLS_PRIVATE(dtls_srtp_info); -#endif /* MBEDTLS_SSL_DTLS_SRTP */ - - /* - * Information for DTLS hello verify - */ -#if defined(MBEDTLS_SSL_DTLS_HELLO_VERIFY) && defined(MBEDTLS_SSL_SRV_C) - unsigned char *MBEDTLS_PRIVATE(cli_id); /*!< transport-level ID of the client */ - size_t MBEDTLS_PRIVATE(cli_id_len); /*!< length of cli_id */ -#endif /* MBEDTLS_SSL_DTLS_HELLO_VERIFY && MBEDTLS_SSL_SRV_C */ - - /* - * Secure renegotiation - */ - /* needed to know when to send extension on server */ - int MBEDTLS_PRIVATE(secure_renegotiation); /*!< does peer support legacy or - secure renegotiation */ -#if defined(MBEDTLS_SSL_RENEGOTIATION) - size_t MBEDTLS_PRIVATE(verify_data_len); /*!< length of verify data stored */ - char MBEDTLS_PRIVATE(own_verify_data)[MBEDTLS_SSL_VERIFY_DATA_MAX_LEN]; /*!< previous handshake verify data */ - char MBEDTLS_PRIVATE(peer_verify_data)[MBEDTLS_SSL_VERIFY_DATA_MAX_LEN]; /*!< previous handshake verify data */ -#endif /* MBEDTLS_SSL_RENEGOTIATION */ - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - /* CID configuration to use in subsequent handshakes. */ - - /*! The next incoming CID, chosen by the user and applying to - * all subsequent handshakes. This may be different from the - * CID currently used in case the user has re-configured the CID - * after an initial handshake. */ - unsigned char MBEDTLS_PRIVATE(own_cid)[MBEDTLS_SSL_CID_IN_LEN_MAX]; - uint8_t MBEDTLS_PRIVATE(own_cid_len); /*!< The length of \c own_cid. */ - uint8_t MBEDTLS_PRIVATE(negotiate_cid); /*!< This indicates whether the CID extension should - * be negotiated in the next handshake or not. - * Possible values are #MBEDTLS_SSL_CID_ENABLED - * and #MBEDTLS_SSL_CID_DISABLED. */ -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - - /** Callback to export key block and master secret */ - mbedtls_ssl_export_keys_t *MBEDTLS_PRIVATE(f_export_keys); - void *MBEDTLS_PRIVATE(p_export_keys); /*!< context for key export callback */ - - /** User data pointer or handle. - * - * The library sets this to \p 0 when creating a context and does not - * access it afterwards. - * - * \warning Serializing and restoring an SSL context with - * mbedtls_ssl_context_save() and mbedtls_ssl_context_load() - * does not currently restore the user data. - */ - mbedtls_ssl_user_data_t MBEDTLS_PRIVATE(user_data); -}; - -/** - * \brief Return the name of the ciphersuite associated with the - * given ID - * - * \param ciphersuite_id SSL ciphersuite ID - * - * \return a string containing the ciphersuite name - */ -const char *mbedtls_ssl_get_ciphersuite_name(const int ciphersuite_id); - -/** - * \brief Return the ID of the ciphersuite associated with the - * given name - * - * \param ciphersuite_name SSL ciphersuite name - * - * \return the ID with the ciphersuite or 0 if not found - */ -int mbedtls_ssl_get_ciphersuite_id(const char *ciphersuite_name); - -/** - * \brief Initialize an SSL context - * Just makes the context ready for mbedtls_ssl_setup() or - * mbedtls_ssl_free() - * - * \param ssl SSL context - */ -void mbedtls_ssl_init(mbedtls_ssl_context *ssl); - -/** - * \brief Set up an SSL context for use - * - * \note No copy of the configuration context is made, it can be - * shared by many mbedtls_ssl_context structures. - * - * \warning The conf structure will be accessed during the session. - * It must not be modified or freed as long as the session - * is active. - * - * \warning This function must be called exactly once per context. - * Calling mbedtls_ssl_setup again is not supported, even - * if no session is active. - * - * \warning After setting up a client context, if certificate-based - * authentication is enabled, you should call - * mbedtls_ssl_set_hostname() to specifiy the expected - * name of the server. Without this, in most scenarios, - * the TLS connection is insecure. See - * #MBEDTLS_ERR_SSL_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * for more information. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param ssl SSL context - * \param conf SSL configuration to use - * - * \return 0 if successful, or MBEDTLS_ERR_SSL_ALLOC_FAILED if - * memory allocation failed - */ -int mbedtls_ssl_setup(mbedtls_ssl_context *ssl, - const mbedtls_ssl_config *conf); - -/** - * \brief Reset an already initialized SSL context for re-use - * while retaining application-set variables, function - * pointers and data. - * - * \param ssl SSL context - * \return 0 if successful, or MBEDTLS_ERR_SSL_ALLOC_FAILED or - MBEDTLS_ERR_SSL_HW_ACCEL_FAILED - */ -int mbedtls_ssl_session_reset(mbedtls_ssl_context *ssl); - -/** - * \brief Set the current endpoint type - * - * \param conf SSL configuration - * \param endpoint must be MBEDTLS_SSL_IS_CLIENT or MBEDTLS_SSL_IS_SERVER - */ -void mbedtls_ssl_conf_endpoint(mbedtls_ssl_config *conf, int endpoint); - -/** - * \brief Get the current endpoint type - * - * \param conf SSL configuration - * - * \return Endpoint type, either MBEDTLS_SSL_IS_CLIENT - * or MBEDTLS_SSL_IS_SERVER - */ -static inline int mbedtls_ssl_conf_get_endpoint(const mbedtls_ssl_config *conf) -{ - return conf->MBEDTLS_PRIVATE(endpoint); -} - -/** - * \brief Set the transport type (TLS or DTLS). - * Default: TLS - * - * \note For DTLS, you must either provide a recv callback that - * doesn't block, or one that handles timeouts, see - * \c mbedtls_ssl_set_bio(). You also need to provide timer - * callbacks with \c mbedtls_ssl_set_timer_cb(). - * - * \param conf SSL configuration - * \param transport transport type: - * MBEDTLS_SSL_TRANSPORT_STREAM for TLS, - * MBEDTLS_SSL_TRANSPORT_DATAGRAM for DTLS. - */ -void mbedtls_ssl_conf_transport(mbedtls_ssl_config *conf, int transport); - -/** - * \brief Set the certificate verification mode - * Default: NONE on server, REQUIRED on client - * - * \param conf SSL configuration - * \param authmode can be: - * - * MBEDTLS_SSL_VERIFY_NONE: peer certificate is not checked - * (default on server) - * (insecure on client) - * - * MBEDTLS_SSL_VERIFY_OPTIONAL: peer certificate is checked, however the - * handshake continues even if verification failed; - * mbedtls_ssl_get_verify_result() can be called after the - * handshake is complete. - * - * MBEDTLS_SSL_VERIFY_REQUIRED: peer *must* present a valid certificate, - * handshake is aborted if verification failed. - * (default on client) - * - * \note On client, MBEDTLS_SSL_VERIFY_REQUIRED is the recommended mode. - * With MBEDTLS_SSL_VERIFY_OPTIONAL, the user needs to call mbedtls_ssl_get_verify_result() at - * the right time(s), which may not be obvious, while REQUIRED always perform - * the verification as soon as possible. For example, REQUIRED was protecting - * against the "triple handshake" attack even before it was found. - */ -void mbedtls_ssl_conf_authmode(mbedtls_ssl_config *conf, int authmode); - -#if defined(MBEDTLS_SSL_EARLY_DATA) -/** - * \brief Set the early data mode - * Default: disabled on server and client - * - * \param conf The SSL configuration to use. - * \param early_data_enabled can be: - * - * MBEDTLS_SSL_EARLY_DATA_DISABLED: - * Early data functionality is disabled. This is the default on client and - * server. - * - * MBEDTLS_SSL_EARLY_DATA_ENABLED: - * Early data functionality is enabled and may be negotiated in the handshake. - * Application using early data functionality needs to be aware that the - * security properties for early data (also refered to as 0-RTT data) are - * weaker than those for other kinds of TLS data. See the documentation of - * mbedtls_ssl_write_early_data() and mbedtls_ssl_read_early_data() for more - * information. - * When early data functionality is enabled on server and only in that case, - * the call to one of the APIs that trigger or resume an handshake sequence, - * namely mbedtls_ssl_handshake(), mbedtls_ssl_handshake_step(), - * mbedtls_ssl_read() or mbedtls_ssl_write() may return with the error code - * MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA indicating that some early data have - * been received. To read the early data, call mbedtls_ssl_read_early_data() - * before calling the original function again. - */ -void mbedtls_ssl_conf_early_data(mbedtls_ssl_config *conf, - int early_data_enabled); - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Set the maximum amount of 0-RTT data in bytes - * Default: #MBEDTLS_SSL_MAX_EARLY_DATA_SIZE - * - * This function sets the value of the max_early_data_size - * field of the early data indication extension included in - * the NewSessionTicket messages that the server may send. - * - * The value defines the maximum amount of 0-RTT data - * in bytes that a client will be allowed to send when using - * one of the tickets defined by the NewSessionTicket messages. - * - * \note When resuming a session using a ticket, if the server receives more - * early data than allowed for the ticket, it terminates the connection. - * The maximum amount of 0-RTT data should thus be large enough - * to allow a minimum of early data to be exchanged. - * - * \param[in] conf The SSL configuration to use. - * \param[in] max_early_data_size The maximum amount of 0-RTT data. - * - * \warning This interface DOES NOT influence/limit the amount of early data - * that can be received through previously created and issued tickets, - * which clients may have stored. - */ -void mbedtls_ssl_conf_max_early_data_size( - mbedtls_ssl_config *conf, uint32_t max_early_data_size); -#endif /* MBEDTLS_SSL_SRV_C */ - -#endif /* MBEDTLS_SSL_EARLY_DATA */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Set the verification callback (Optional). - * - * If set, the provided verify callback is called for each - * certificate in the peer's CRT chain, including the trusted - * root. For more information, please see the documentation of - * \c mbedtls_x509_crt_verify(). - * - * \note For per context callbacks and contexts, please use - * mbedtls_ssl_set_verify() instead. - * - * \param conf The SSL configuration to use. - * \param f_vrfy The verification callback to use during CRT verification. - * \param p_vrfy The opaque context to be passed to the callback. - */ -void mbedtls_ssl_conf_verify(mbedtls_ssl_config *conf, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -/** - * \brief Set the random number generator callback - * - * \note The callback with its parameter must remain valid as - * long as there is an SSL context that uses the - * SSL configuration. - * - * \param conf SSL configuration - * \param f_rng RNG function (mandatory) - * \param p_rng RNG parameter - */ -void mbedtls_ssl_conf_rng(mbedtls_ssl_config *conf, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Set the debug callback - * - * The callback has the following argument: - * void * opaque context for the callback - * int debug level - * const char * file name - * int line number - * const char * message - * - * \param conf SSL configuration - * \param f_dbg debug function - * \param p_dbg debug parameter - */ -void mbedtls_ssl_conf_dbg(mbedtls_ssl_config *conf, - void (*f_dbg)(void *, int, const char *, int, const char *), - void *p_dbg); - -/** - * \brief Return the SSL configuration structure associated - * with the given SSL context. - * - * \note The pointer returned by this function is guaranteed to - * remain valid until the context is freed. - * - * \param ssl The SSL context to query. - * \return Pointer to the SSL configuration associated with \p ssl. - */ -static inline const mbedtls_ssl_config *mbedtls_ssl_context_get_config( - const mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(conf); -} - -/** - * \brief Set the underlying BIO callbacks for write, read and - * read-with-timeout. - * - * \param ssl SSL context - * \param p_bio parameter (context) shared by BIO callbacks - * \param f_send write callback - * \param f_recv read callback - * \param f_recv_timeout blocking read callback with timeout. - * - * \note One of f_recv or f_recv_timeout can be NULL, in which case - * the other is used. If both are non-NULL, f_recv_timeout is - * used and f_recv is ignored (as if it were NULL). - * - * \note The two most common use cases are: - * - non-blocking I/O, f_recv != NULL, f_recv_timeout == NULL - * - blocking I/O, f_recv == NULL, f_recv_timeout != NULL - * - * \note For DTLS, you need to provide either a non-NULL - * f_recv_timeout callback, or a f_recv that doesn't block. - * - * \note See the documentations of \c mbedtls_ssl_send_t, - * \c mbedtls_ssl_recv_t and \c mbedtls_ssl_recv_timeout_t for - * the conventions those callbacks must follow. - * - * \note On some platforms, net_sockets.c provides - * \c mbedtls_net_send(), \c mbedtls_net_recv() and - * \c mbedtls_net_recv_timeout() that are suitable to be used - * here. - */ -void mbedtls_ssl_set_bio(mbedtls_ssl_context *ssl, - void *p_bio, - mbedtls_ssl_send_t *f_send, - mbedtls_ssl_recv_t *f_recv, - mbedtls_ssl_recv_timeout_t *f_recv_timeout); - -#if defined(MBEDTLS_SSL_PROTO_DTLS) - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) - - -/** - * \brief Configure the use of the Connection ID (CID) - * extension in the next handshake. - * - * Reference: RFC 9146 (or draft-ietf-tls-dtls-connection-id-05 - * https://tools.ietf.org/html/draft-ietf-tls-dtls-connection-id-05 - * for legacy version) - * - * The DTLS CID extension allows the reliable association of - * DTLS records to DTLS connections across changes in the - * underlying transport (changed IP and Port metadata) by - * adding explicit connection identifiers (CIDs) to the - * headers of encrypted DTLS records. The desired CIDs are - * configured by the application layer and are exchanged in - * new `ClientHello` / `ServerHello` extensions during the - * handshake, where each side indicates the CID it wants the - * peer to use when writing encrypted messages. The CIDs are - * put to use once records get encrypted: the stack discards - * any incoming records that don't include the configured CID - * in their header, and adds the peer's requested CID to the - * headers of outgoing messages. - * - * This API enables or disables the use of the CID extension - * in the next handshake and sets the value of the CID to - * be used for incoming messages. - * - * \param ssl The SSL context to configure. This must be initialized. - * \param enable This value determines whether the CID extension should - * be used or not. Possible values are: - * - MBEDTLS_SSL_CID_ENABLED to enable the use of the CID. - * - MBEDTLS_SSL_CID_DISABLED (default) to disable the use - * of the CID. - * \param own_cid The address of the readable buffer holding the CID we want - * the peer to use when sending encrypted messages to us. - * This may be \c NULL if \p own_cid_len is \c 0. - * This parameter is unused if \p enable is set to - * MBEDTLS_SSL_CID_DISABLED. - * \param own_cid_len The length of \p own_cid. - * This parameter is unused if \p enable is set to - * MBEDTLS_SSL_CID_DISABLED. - * - * \note The value of \p own_cid_len must match the value of the - * \c len parameter passed to mbedtls_ssl_conf_cid() - * when configuring the ::mbedtls_ssl_config that \p ssl - * is bound to. - * - * \note This CID configuration applies to subsequent handshakes - * performed on the SSL context \p ssl, but does not trigger - * one. You still have to call `mbedtls_ssl_handshake()` - * (for the initial handshake) or `mbedtls_ssl_renegotiate()` - * (for a renegotiation handshake) explicitly after a - * successful call to this function to run the handshake. - * - * \note This call cannot guarantee that the use of the CID - * will be successfully negotiated in the next handshake, - * because the peer might not support it. Specifically: - * - On the Client, enabling the use of the CID through - * this call implies that the `ClientHello` in the next - * handshake will include the CID extension, thereby - * offering the use of the CID to the server. Only if - * the `ServerHello` contains the CID extension, too, - * the CID extension will actually be put to use. - * - On the Server, enabling the use of the CID through - * this call implies that the server will look for - * the CID extension in a `ClientHello` from the client, - * and, if present, reply with a CID extension in its - * `ServerHello`. - * - * \note To check whether the use of the CID was negotiated - * after the subsequent handshake has completed, please - * use the API mbedtls_ssl_get_peer_cid(). - * - * \warning If the use of the CID extension is enabled in this call - * and the subsequent handshake negotiates its use, Mbed TLS - * will silently drop every packet whose CID does not match - * the CID configured in \p own_cid. It is the responsibility - * of the user to adapt the underlying transport to take care - * of CID-based demultiplexing before handing datagrams to - * Mbed TLS. - * - * \return \c 0 on success. In this case, the CID configuration - * applies to the next handshake. - * \return A negative error code on failure. - */ -int mbedtls_ssl_set_cid(mbedtls_ssl_context *ssl, - int enable, - unsigned char const *own_cid, - size_t own_cid_len); - -/** - * \brief Get information about our request for usage of the CID - * extension in the current connection. - * - * \param ssl The SSL context to query. - * \param enabled The address at which to store whether the CID extension - * is requested to be used or not. If the CID is - * requested, `*enabled` is set to - * MBEDTLS_SSL_CID_ENABLED; otherwise, it is set to - * MBEDTLS_SSL_CID_DISABLED. - * \param own_cid The address of the buffer in which to store our own - * CID (if the CID extension is requested). This may be - * \c NULL in case the value of our CID isn't needed. If - * it is not \c NULL, \p own_cid_len must not be \c NULL. - * \param own_cid_len The address at which to store the size of our own CID - * (if the CID extension is requested). This is also the - * number of Bytes in \p own_cid that have been written. - * This may be \c NULL in case the length of our own CID - * isn't needed. If it is \c NULL, \p own_cid must be - * \c NULL, too. - * - *\note If we are requesting an empty CID this function sets - * `*enabled` to #MBEDTLS_SSL_CID_DISABLED (the rationale - * for this is that the resulting outcome is the - * same as if the CID extensions wasn't requested). - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_ssl_get_own_cid(mbedtls_ssl_context *ssl, - int *enabled, - unsigned char own_cid[MBEDTLS_SSL_CID_IN_LEN_MAX], - size_t *own_cid_len); - -/** - * \brief Get information about the use of the CID extension - * in the current connection. - * - * \param ssl The SSL context to query. - * \param enabled The address at which to store whether the CID extension - * is currently in use or not. If the CID is in use, - * `*enabled` is set to MBEDTLS_SSL_CID_ENABLED; - * otherwise, it is set to MBEDTLS_SSL_CID_DISABLED. - * \param peer_cid The address of the buffer in which to store the CID - * chosen by the peer (if the CID extension is used). - * This may be \c NULL in case the value of peer CID - * isn't needed. If it is not \c NULL, \p peer_cid_len - * must not be \c NULL. - * \param peer_cid_len The address at which to store the size of the CID - * chosen by the peer (if the CID extension is used). - * This is also the number of Bytes in \p peer_cid that - * have been written. - * This may be \c NULL in case the length of the peer CID - * isn't needed. If it is \c NULL, \p peer_cid must be - * \c NULL, too. - * - * \note This applies to the state of the CID negotiated in - * the last complete handshake. If a handshake is in - * progress, this function will attempt to complete - * the handshake first. - * - * \note If CID extensions have been exchanged but both client - * and server chose to use an empty CID, this function - * sets `*enabled` to #MBEDTLS_SSL_CID_DISABLED - * (the rationale for this is that the resulting - * communication is the same as if the CID extensions - * hadn't been used). - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_ssl_get_peer_cid(mbedtls_ssl_context *ssl, - int *enabled, - unsigned char peer_cid[MBEDTLS_SSL_CID_OUT_LEN_MAX], - size_t *peer_cid_len); - -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - -/** - * \brief Set the Maximum Transport Unit (MTU). - * Special value: 0 means unset (no limit). - * This represents the maximum size of a datagram payload - * handled by the transport layer (usually UDP) as determined - * by the network link and stack. In practice, this controls - * the maximum size datagram the DTLS layer will pass to the - * \c f_send() callback set using \c mbedtls_ssl_set_bio(). - * - * \note The limit on datagram size is converted to a limit on - * record payload by subtracting the current overhead of - * encapsulation and encryption/authentication if any. - * - * \note This can be called at any point during the connection, for - * example when a Path Maximum Transfer Unit (PMTU) - * estimate becomes available from other sources, - * such as lower (or higher) protocol layers. - * - * \note This setting only controls the size of the packets we send, - * and does not restrict the size of the datagrams we're - * willing to receive. Client-side, you can request the - * server to use smaller records with \c - * mbedtls_ssl_conf_max_frag_len(). - * - * \note If both a MTU and a maximum fragment length have been - * configured (or negotiated with the peer), the resulting - * lower limit on record payload (see first note) is used. - * - * \note This can only be used to decrease the maximum size - * of datagrams (hence records, see first note) sent. It - * cannot be used to increase the maximum size of records over - * the limit set by #MBEDTLS_SSL_OUT_CONTENT_LEN. - * - * \note Values lower than the current record layer expansion will - * result in an error when trying to send data. - * - * \param ssl SSL context - * \param mtu Value of the path MTU in bytes - */ -void mbedtls_ssl_set_mtu(mbedtls_ssl_context *ssl, uint16_t mtu); -#endif /* MBEDTLS_SSL_PROTO_DTLS */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Set a connection-specific verification callback (optional). - * - * If set, the provided verify callback is called for each - * certificate in the peer's CRT chain, including the trusted - * root. For more information, please see the documentation of - * \c mbedtls_x509_crt_verify(). - * - * \note This call is analogous to mbedtls_ssl_conf_verify() but - * binds the verification callback and context to an SSL context - * as opposed to an SSL configuration. - * If mbedtls_ssl_conf_verify() and mbedtls_ssl_set_verify() - * are both used, mbedtls_ssl_set_verify() takes precedence. - * - * \param ssl The SSL context to use. - * \param f_vrfy The verification callback to use during CRT verification. - * \param p_vrfy The opaque context to be passed to the callback. - */ -void mbedtls_ssl_set_verify(mbedtls_ssl_context *ssl, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -/** - * \brief Set the timeout period for mbedtls_ssl_read() - * (Default: no timeout.) - * - * \param conf SSL configuration context - * \param timeout Timeout value in milliseconds. - * Use 0 for no timeout (default). - * - * \note With blocking I/O, this will only work if a non-NULL - * \c f_recv_timeout was set with \c mbedtls_ssl_set_bio(). - * With non-blocking I/O, this will only work if timer - * callbacks were set with \c mbedtls_ssl_set_timer_cb(). - * - * \note With non-blocking I/O, you may also skip this function - * altogether and handle timeouts at the application layer. - */ -void mbedtls_ssl_conf_read_timeout(mbedtls_ssl_config *conf, uint32_t timeout); - -/** - * \brief Check whether a buffer contains a valid and authentic record - * that has not been seen before. (DTLS only). - * - * This function does not change the user-visible state - * of the SSL context. Its sole purpose is to provide - * an indication of the legitimacy of an incoming record. - * - * This can be useful e.g. in distributed server environments - * using the DTLS Connection ID feature, in which connections - * might need to be passed between service instances on a change - * of peer address, but where such disruptive operations should - * only happen after the validity of incoming records has been - * confirmed. - * - * \param ssl The SSL context to use. - * \param buf The address of the buffer holding the record to be checked. - * This must be a read/write buffer of length \p buflen Bytes. - * \param buflen The length of \p buf in Bytes. - * - * \note This routine only checks whether the provided buffer begins - * with a valid and authentic record that has not been seen - * before, but does not check potential data following the - * initial record. In particular, it is possible to pass DTLS - * datagrams containing multiple records, in which case only - * the first record is checked. - * - * \note This function modifies the input buffer \p buf. If you need - * to preserve the original record, you have to maintain a copy. - * - * \return \c 0 if the record is valid and authentic and has not been - * seen before. - * \return MBEDTLS_ERR_SSL_INVALID_MAC if the check completed - * successfully but the record was found to be not authentic. - * \return MBEDTLS_ERR_SSL_INVALID_RECORD if the check completed - * successfully but the record was found to be invalid for - * a reason different from authenticity checking. - * \return MBEDTLS_ERR_SSL_UNEXPECTED_RECORD if the check completed - * successfully but the record was found to be unexpected - * in the state of the SSL context, including replayed records. - * \return Another negative error code on different kinds of failure. - * In this case, the SSL context becomes unusable and needs - * to be freed or reset before reuse. - */ -int mbedtls_ssl_check_record(mbedtls_ssl_context const *ssl, - unsigned char *buf, - size_t buflen); - -/** - * \brief Set the timer callbacks (Mandatory for DTLS.) - * - * \param ssl SSL context - * \param p_timer parameter (context) shared by timer callbacks - * \param f_set_timer set timer callback - * \param f_get_timer get timer callback. Must return: - * - * \note See the documentation of \c mbedtls_ssl_set_timer_t and - * \c mbedtls_ssl_get_timer_t for the conventions this pair of - * callbacks must follow. - * - * \note On some platforms, timing.c provides - * \c mbedtls_timing_set_delay() and - * \c mbedtls_timing_get_delay() that are suitable for using - * here, except if using an event-driven style. - * - * \note See also the "DTLS tutorial" article in our knowledge base. - * https://mbed-tls.readthedocs.io/en/latest/kb/how-to/dtls-tutorial - */ -void mbedtls_ssl_set_timer_cb(mbedtls_ssl_context *ssl, - void *p_timer, - mbedtls_ssl_set_timer_t *f_set_timer, - mbedtls_ssl_get_timer_t *f_get_timer); - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Set the certificate selection callback (server-side only). - * - * If set, the callback is always called for each handshake, - * after `ClientHello` processing has finished. - * - * \param conf The SSL configuration to register the callback with. - * \param f_cert_cb The callback for selecting server certificate after - * `ClientHello` processing has finished. - */ -static inline void mbedtls_ssl_conf_cert_cb(mbedtls_ssl_config *conf, - mbedtls_ssl_hs_cb_t f_cert_cb) -{ - conf->MBEDTLS_PRIVATE(f_cert_cb) = f_cert_cb; -} -#endif /* MBEDTLS_SSL_SRV_C */ - -/** - * \brief Callback type: generate and write session ticket - * - * \note This describes what a callback implementation should do. - * This callback should generate an encrypted and - * authenticated ticket for the session and write it to the - * output buffer. Here, ticket means the opaque ticket part - * of the NewSessionTicket structure of RFC 5077. - * - * \param p_ticket Context for the callback - * \param session SSL session to be written in the ticket - * \param start Start of the output buffer - * \param end End of the output buffer - * \param tlen On exit, holds the length written - * \param lifetime On exit, holds the lifetime of the ticket in seconds - * - * \return 0 if successful, or - * a specific MBEDTLS_ERR_XXX code. - */ -typedef int mbedtls_ssl_ticket_write_t(void *p_ticket, - const mbedtls_ssl_session *session, - unsigned char *start, - const unsigned char *end, - size_t *tlen, - uint32_t *lifetime); - -/** - * \brief Callback type: parse and load session ticket - * - * \note This describes what a callback implementation should do. - * This callback should parse a session ticket as generated - * by the corresponding mbedtls_ssl_ticket_write_t function, - * and, if the ticket is authentic and valid, load the - * session. - * - * \note The implementation is allowed to modify the first len - * bytes of the input buffer, eg to use it as a temporary - * area for the decrypted ticket contents. - * - * \param p_ticket Context for the callback - * \param session SSL session to be loaded - * \param buf Start of the buffer containing the ticket - * \param len Length of the ticket. - * - * \return 0 if successful, or - * MBEDTLS_ERR_SSL_INVALID_MAC if not authentic, or - * MBEDTLS_ERR_SSL_SESSION_TICKET_EXPIRED if expired, or - * any other non-zero code for other failures. - */ -typedef int mbedtls_ssl_ticket_parse_t(void *p_ticket, - mbedtls_ssl_session *session, - unsigned char *buf, - size_t len); - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Configure SSL session ticket callbacks (server only). - * (Default: none.) - * - * \note On server, session tickets are enabled by providing - * non-NULL callbacks. - * - * \note On client, use \c mbedtls_ssl_conf_session_tickets(). - * - * \param conf SSL configuration context - * \param f_ticket_write Callback for writing a ticket - * \param f_ticket_parse Callback for parsing a ticket - * \param p_ticket Context shared by the two callbacks - */ -void mbedtls_ssl_conf_session_tickets_cb(mbedtls_ssl_config *conf, - mbedtls_ssl_ticket_write_t *f_ticket_write, - mbedtls_ssl_ticket_parse_t *f_ticket_parse, - void *p_ticket); - -#if defined(MBEDTLS_HAVE_TIME) -/** - * \brief Get the creation time of a session ticket. - * - * \note See the documentation of \c ticket_creation_time for information about - * the intended usage of this function. - * - * \param session SSL session - * \param ticket_creation_time On exit, holds the ticket creation time in - * milliseconds. - * - * \return 0 on success, - * MBEDTLS_ERR_SSL_BAD_INPUT_DATA if an input is not valid. - */ -static inline int mbedtls_ssl_session_get_ticket_creation_time( - mbedtls_ssl_session *session, mbedtls_ms_time_t *ticket_creation_time) -{ - if (session == NULL || ticket_creation_time == NULL || - session->MBEDTLS_PRIVATE(endpoint) != MBEDTLS_SSL_IS_SERVER) { - return MBEDTLS_ERR_SSL_BAD_INPUT_DATA; - } - - *ticket_creation_time = session->MBEDTLS_PRIVATE(ticket_creation_time); - - return 0; -} -#endif /* MBEDTLS_HAVE_TIME */ -#endif /* MBEDTLS_SSL_SESSION_TICKETS && MBEDTLS_SSL_SRV_C */ - -/** - * \brief Get the session-id buffer. - * - * \param session SSL session. - * - * \return The address of the session-id buffer. - */ -static inline unsigned const char (*mbedtls_ssl_session_get_id(const mbedtls_ssl_session * - session))[32] -{ - return &session->MBEDTLS_PRIVATE(id); -} - -/** - * \brief Get the size of the session-id. - * - * \param session SSL session. - * - * \return size_t size of session-id buffer. - */ -static inline size_t mbedtls_ssl_session_get_id_len(const mbedtls_ssl_session *session) -{ - return session->MBEDTLS_PRIVATE(id_len); -} - -/** - * \brief Get the ciphersuite-id. - * - * \param session SSL session. - * - * \return int represetation for ciphersuite. - */ -static inline int mbedtls_ssl_session_get_ciphersuite_id(const mbedtls_ssl_session *session) -{ - return session->MBEDTLS_PRIVATE(ciphersuite); -} - -/** - * \brief Configure a key export callback. - * (Default: none.) - * - * This API can be used for two purposes: - * - Debugging: Use this API to e.g. generate an NSSKeylog - * file and use it to inspect encrypted traffic in tools - * such as Wireshark. - * - Application-specific export: Use this API to implement - * key exporters, e.g. for EAP-TLS or DTLS-SRTP. - * - * - * \param ssl The SSL context to which the export - * callback should be attached. - * \param f_export_keys The callback for the key export. - * \param p_export_keys The opaque context pointer to be passed to the - * callback \p f_export_keys. - */ -void mbedtls_ssl_set_export_keys_cb(mbedtls_ssl_context *ssl, - mbedtls_ssl_export_keys_t *f_export_keys, - void *p_export_keys); - -/** \brief Set the user data in an SSL configuration to a pointer. - * - * You can retrieve this value later with mbedtls_ssl_conf_get_user_data_p(). - * - * \note The library stores \c p without accessing it. It is the responsibility - * of the caller to ensure that the pointer remains valid. - * - * \param conf The SSL configuration context to modify. - * \param p The new value of the user data. - */ -static inline void mbedtls_ssl_conf_set_user_data_p( - mbedtls_ssl_config *conf, - void *p) -{ - conf->MBEDTLS_PRIVATE(user_data).p = p; -} - -/** \brief Set the user data in an SSL configuration to an integer. - * - * You can retrieve this value later with mbedtls_ssl_conf_get_user_data_n(). - * - * \param conf The SSL configuration context to modify. - * \param n The new value of the user data. - */ -static inline void mbedtls_ssl_conf_set_user_data_n( - mbedtls_ssl_config *conf, - uintptr_t n) -{ - conf->MBEDTLS_PRIVATE(user_data).n = n; -} - -/** \brief Retrieve the user data in an SSL configuration as a pointer. - * - * This is the value last set with mbedtls_ssl_conf_set_user_data_p(), or - * \c NULL if mbedtls_ssl_conf_set_user_data_p() has not previously been - * called. The value is undefined if mbedtls_ssl_conf_set_user_data_n() has - * been called without a subsequent call to mbedtls_ssl_conf_set_user_data_p(). - * - * \param conf The SSL configuration context to modify. - * \return The current value of the user data. - */ -static inline void *mbedtls_ssl_conf_get_user_data_p( - mbedtls_ssl_config *conf) -{ - return conf->MBEDTLS_PRIVATE(user_data).p; -} - -/** \brief Retrieve the user data in an SSL configuration as an integer. - * - * This is the value last set with mbedtls_ssl_conf_set_user_data_n(), or - * \c 0 if mbedtls_ssl_conf_set_user_data_n() has not previously been - * called. The value is undefined if mbedtls_ssl_conf_set_user_data_p() has - * been called without a subsequent call to mbedtls_ssl_conf_set_user_data_n(). - * - * \param conf The SSL configuration context to modify. - * \return The current value of the user data. - */ -static inline uintptr_t mbedtls_ssl_conf_get_user_data_n( - mbedtls_ssl_config *conf) -{ - return conf->MBEDTLS_PRIVATE(user_data).n; -} - -/** \brief Set the user data in an SSL context to a pointer. - * - * You can retrieve this value later with mbedtls_ssl_get_user_data_p(). - * - * \note The library stores \c p without accessing it. It is the responsibility - * of the caller to ensure that the pointer remains valid. - * - * \param ssl The SSL context to modify. - * \param p The new value of the user data. - */ -static inline void mbedtls_ssl_set_user_data_p( - mbedtls_ssl_context *ssl, - void *p) -{ - ssl->MBEDTLS_PRIVATE(user_data).p = p; -} - -/** \brief Set the user data in an SSL context to an integer. - * - * You can retrieve this value later with mbedtls_ssl_get_user_data_n(). - * - * \param ssl The SSL context to modify. - * \param n The new value of the user data. - */ -static inline void mbedtls_ssl_set_user_data_n( - mbedtls_ssl_context *ssl, - uintptr_t n) -{ - ssl->MBEDTLS_PRIVATE(user_data).n = n; -} - -/** \brief Retrieve the user data in an SSL context as a pointer. - * - * This is the value last set with mbedtls_ssl_set_user_data_p(), or - * \c NULL if mbedtls_ssl_set_user_data_p() has not previously been - * called. The value is undefined if mbedtls_ssl_set_user_data_n() has - * been called without a subsequent call to mbedtls_ssl_set_user_data_p(). - * - * \param ssl The SSL context to modify. - * \return The current value of the user data. - */ -static inline void *mbedtls_ssl_get_user_data_p( - mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(user_data).p; -} - -/** \brief Retrieve the user data in an SSL context as an integer. - * - * This is the value last set with mbedtls_ssl_set_user_data_n(), or - * \c 0 if mbedtls_ssl_set_user_data_n() has not previously been - * called. The value is undefined if mbedtls_ssl_set_user_data_p() has - * been called without a subsequent call to mbedtls_ssl_set_user_data_n(). - * - * \param ssl The SSL context to modify. - * \return The current value of the user data. - */ -static inline uintptr_t mbedtls_ssl_get_user_data_n( - mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(user_data).n; -} - -#if defined(MBEDTLS_SSL_ASYNC_PRIVATE) -/** - * \brief Configure asynchronous private key operation callbacks. - * - * \param conf SSL configuration context - * \param f_async_sign Callback to start a signature operation. See - * the description of ::mbedtls_ssl_async_sign_t - * for more information. This may be \c NULL if the - * external processor does not support any signature - * operation; in this case the private key object - * associated with the certificate will be used. - * \param f_async_decrypt Callback to start a decryption operation. See - * the description of ::mbedtls_ssl_async_decrypt_t - * for more information. This may be \c NULL if the - * external processor does not support any decryption - * operation; in this case the private key object - * associated with the certificate will be used. - * \param f_async_resume Callback to resume an asynchronous operation. See - * the description of ::mbedtls_ssl_async_resume_t - * for more information. This may not be \c NULL unless - * \p f_async_sign and \p f_async_decrypt are both - * \c NULL. - * \param f_async_cancel Callback to cancel an asynchronous operation. See - * the description of ::mbedtls_ssl_async_cancel_t - * for more information. This may be \c NULL if - * no cleanup is needed. - * \param config_data A pointer to configuration data which can be - * retrieved with - * mbedtls_ssl_conf_get_async_config_data(). The - * library stores this value without dereferencing it. - */ -void mbedtls_ssl_conf_async_private_cb(mbedtls_ssl_config *conf, - mbedtls_ssl_async_sign_t *f_async_sign, - mbedtls_ssl_async_decrypt_t *f_async_decrypt, - mbedtls_ssl_async_resume_t *f_async_resume, - mbedtls_ssl_async_cancel_t *f_async_cancel, - void *config_data); - -/** - * \brief Retrieve the configuration data set by - * mbedtls_ssl_conf_async_private_cb(). - * - * \param conf SSL configuration context - * \return The configuration data set by - * mbedtls_ssl_conf_async_private_cb(). - */ -void *mbedtls_ssl_conf_get_async_config_data(const mbedtls_ssl_config *conf); - -/** - * \brief Retrieve the asynchronous operation user context. - * - * \note This function may only be called while a handshake - * is in progress. - * - * \param ssl The SSL context to access. - * - * \return The asynchronous operation user context that was last - * set during the current handshake. If - * mbedtls_ssl_set_async_operation_data() has not yet been - * called during the current handshake, this function returns - * \c NULL. - */ -void *mbedtls_ssl_get_async_operation_data(const mbedtls_ssl_context *ssl); - -/** - * \brief Retrieve the asynchronous operation user context. - * - * \note This function may only be called while a handshake - * is in progress. - * - * \param ssl The SSL context to access. - * \param ctx The new value of the asynchronous operation user context. - * Call mbedtls_ssl_get_async_operation_data() later during the - * same handshake to retrieve this value. - */ -void mbedtls_ssl_set_async_operation_data(mbedtls_ssl_context *ssl, - void *ctx); -#endif /* MBEDTLS_SSL_ASYNC_PRIVATE */ - -/** - * \brief Callback type: generate a cookie - * - * \param ctx Context for the callback - * \param p Buffer to write to, - * must be updated to point right after the cookie - * \param end Pointer to one past the end of the output buffer - * \param info Client ID info that was passed to - * \c mbedtls_ssl_set_client_transport_id() - * \param ilen Length of info in bytes - * - * \return The callback must return 0 on success, - * or a negative error code. - */ -typedef int mbedtls_ssl_cookie_write_t(void *ctx, - unsigned char **p, unsigned char *end, - const unsigned char *info, size_t ilen); - -/** - * \brief Callback type: verify a cookie - * - * \param ctx Context for the callback - * \param cookie Cookie to verify - * \param clen Length of cookie - * \param info Client ID info that was passed to - * \c mbedtls_ssl_set_client_transport_id() - * \param ilen Length of info in bytes - * - * \return The callback must return 0 if cookie is valid, - * or a negative error code. - */ -typedef int mbedtls_ssl_cookie_check_t(void *ctx, - const unsigned char *cookie, size_t clen, - const unsigned char *info, size_t ilen); - -#if defined(MBEDTLS_SSL_DTLS_HELLO_VERIFY) && defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Register callbacks for DTLS cookies - * (Server only. DTLS only.) - * - * Default: dummy callbacks that fail, in order to force you to - * register working callbacks (and initialize their context). - * - * To disable HelloVerifyRequest, register NULL callbacks. - * - * \warning Disabling hello verification allows your server to be used - * for amplification in DoS attacks against other hosts. - * Only disable if you known this can't happen in your - * particular environment. - * - * \note See comments on \c mbedtls_ssl_handshake() about handling - * the MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED that is expected - * on the first handshake attempt when this is enabled. - * - * \note This is also necessary to handle client reconnection from - * the same port as described in RFC 6347 section 4.2.8 (only - * the variant with cookies is supported currently). See - * comments on \c mbedtls_ssl_read() for details. - * - * \param conf SSL configuration - * \param f_cookie_write Cookie write callback - * \param f_cookie_check Cookie check callback - * \param p_cookie Context for both callbacks - */ -void mbedtls_ssl_conf_dtls_cookies(mbedtls_ssl_config *conf, - mbedtls_ssl_cookie_write_t *f_cookie_write, - mbedtls_ssl_cookie_check_t *f_cookie_check, - void *p_cookie); - -/** - * \brief Set client's transport-level identification info. - * (Server only. DTLS only.) - * - * This is usually the IP address (and port), but could be - * anything identify the client depending on the underlying - * network stack. Used for HelloVerifyRequest with DTLS. - * This is *not* used to route the actual packets. - * - * \param ssl SSL context - * \param info Transport-level info identifying the client (eg IP + port) - * \param ilen Length of info in bytes - * - * \note An internal copy is made, so the info buffer can be reused. - * - * \return 0 on success, - * MBEDTLS_ERR_SSL_BAD_INPUT_DATA if used on client, - * MBEDTLS_ERR_SSL_ALLOC_FAILED if out of memory. - */ -int mbedtls_ssl_set_client_transport_id(mbedtls_ssl_context *ssl, - const unsigned char *info, - size_t ilen); - -#endif /* MBEDTLS_SSL_DTLS_HELLO_VERIFY && MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY) -/** - * \brief Enable or disable anti-replay protection for DTLS. - * (DTLS only, no effect on TLS.) - * Default: enabled. - * - * \param conf SSL configuration - * \param mode MBEDTLS_SSL_ANTI_REPLAY_ENABLED or MBEDTLS_SSL_ANTI_REPLAY_DISABLED. - * - * \warning Disabling this is a security risk unless the application - * protocol handles duplicated packets in a safe way. You - * should not disable this without careful consideration. - * However, if your application already detects duplicated - * packets and needs information about them to adjust its - * transmission strategy, then you'll want to disable this. - */ -void mbedtls_ssl_conf_dtls_anti_replay(mbedtls_ssl_config *conf, char mode); -#endif /* MBEDTLS_SSL_DTLS_ANTI_REPLAY */ - -/** - * \brief Set a limit on the number of records with a bad MAC - * before terminating the connection. - * (DTLS only, no effect on TLS.) - * Default: 0 (disabled). - * - * \param conf SSL configuration - * \param limit Limit, or 0 to disable. - * - * \note If the limit is N, then the connection is terminated when - * the Nth non-authentic record is seen. - * - * \note Records with an invalid header are not counted, only the - * ones going through the authentication-decryption phase. - * - * \note This is a security trade-off related to the fact that it's - * often relatively easy for an active attacker to inject UDP - * datagrams. On one hand, setting a low limit here makes it - * easier for such an attacker to forcibly terminated a - * connection. On the other hand, a high limit or no limit - * might make us waste resources checking authentication on - * many bogus packets. - */ -void mbedtls_ssl_conf_dtls_badmac_limit(mbedtls_ssl_config *conf, unsigned limit); - -#if defined(MBEDTLS_SSL_PROTO_DTLS) - -/** - * \brief Allow or disallow packing of multiple handshake records - * within a single datagram. - * - * \param ssl The SSL context to configure. - * \param allow_packing This determines whether datagram packing may - * be used or not. A value of \c 0 means that every - * record will be sent in a separate datagram; a - * value of \c 1 means that, if space permits, - * multiple handshake messages (including CCS) belonging to - * a single flight may be packed within a single datagram. - * - * \note This is enabled by default and should only be disabled - * for test purposes, or if datagram packing causes - * interoperability issues with peers that don't support it. - * - * \note Allowing datagram packing reduces the network load since - * there's less overhead if multiple messages share the same - * datagram. Also, it increases the handshake efficiency - * since messages belonging to a single datagram will not - * be reordered in transit, and so future message buffering - * or flight retransmission (if no buffering is used) as - * means to deal with reordering are needed less frequently. - * - * \note Application records are not affected by this option and - * are currently always sent in separate datagrams. - * - */ -void mbedtls_ssl_set_datagram_packing(mbedtls_ssl_context *ssl, - unsigned allow_packing); - -/** - * \brief Set retransmit timeout values for the DTLS handshake. - * (DTLS only, no effect on TLS.) - * - * \param conf SSL configuration - * \param min Initial timeout value in milliseconds. - * Default: 1000 (1 second). - * \param max Maximum timeout value in milliseconds. - * Default: 60000 (60 seconds). - * - * \note Default values are from RFC 6347 section 4.2.4.1. - * - * \note The 'min' value should typically be slightly above the - * expected round-trip time to your peer, plus whatever time - * it takes for the peer to process the message. For example, - * if your RTT is about 600ms and you peer needs up to 1s to - * do the cryptographic operations in the handshake, then you - * should set 'min' slightly above 1600. Lower values of 'min' - * might cause spurious resends which waste network resources, - * while larger value of 'min' will increase overall latency - * on unreliable network links. - * - * \note The more unreliable your network connection is, the larger - * your max / min ratio needs to be in order to achieve - * reliable handshakes. - * - * \note Messages are retransmitted up to log2(ceil(max/min)) times. - * For example, if min = 1s and max = 5s, the retransmit plan - * goes: send ... 1s -> resend ... 2s -> resend ... 4s -> - * resend ... 5s -> give up and return a timeout error. - */ -void mbedtls_ssl_conf_handshake_timeout(mbedtls_ssl_config *conf, uint32_t min, uint32_t max); -#endif /* MBEDTLS_SSL_PROTO_DTLS */ - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Set the session cache callbacks (server-side only) - * If not set, no session resuming is done (except if session - * tickets are enabled too). - * - * The session cache has the responsibility to check for stale - * entries based on timeout. See RFC 5246 for recommendations. - * - * Warning: session.peer_cert is cleared by the SSL/TLS layer on - * connection shutdown, so do not cache the pointer! Either set - * it to NULL or make a full copy of the certificate. - * - * The get callback is called once during the initial handshake - * to enable session resuming. The get function has the - * following parameters: (void *parameter, mbedtls_ssl_session *session) - * If a valid entry is found, it should fill the master of - * the session object with the cached values and return 0, - * return 1 otherwise. Optionally peer_cert can be set as well - * if it is properly present in cache entry. - * - * The set callback is called once during the initial handshake - * to enable session resuming after the entire handshake has - * been finished. The set function has the following parameters: - * (void *parameter, const mbedtls_ssl_session *session). The function - * should create a cache entry for future retrieval based on - * the data in the session structure and should keep in mind - * that the mbedtls_ssl_session object presented (and all its referenced - * data) is cleared by the SSL/TLS layer when the connection is - * terminated. It is recommended to add metadata to determine if - * an entry is still valid in the future. Return 0 if - * successfully cached, return 1 otherwise. - * - * \param conf SSL configuration - * \param p_cache parameter (context) for both callbacks - * \param f_get_cache session get callback - * \param f_set_cache session set callback - */ -void mbedtls_ssl_conf_session_cache(mbedtls_ssl_config *conf, - void *p_cache, - mbedtls_ssl_cache_get_t *f_get_cache, - mbedtls_ssl_cache_set_t *f_set_cache); -#endif /* MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_SSL_CLI_C) -/** - * \brief Load a session for session resumption. - * - * Sessions loaded through this call will be considered - * for session resumption in the next handshake. - * - * \note Even if this call succeeds, it is not guaranteed that - * the next handshake will indeed be shortened through the - * use of session resumption: The server is always free - * to reject any attempt for resumption and fall back to - * a full handshake. - * - * \note This function can handle a variety of mechanisms for session - * resumption: For TLS 1.2, both session ID-based resumption - * and ticket-based resumption will be considered. For TLS 1.3, - * sessions equate to tickets, and loading one session by - * calling this function will lead to its corresponding ticket - * being advertised as resumption PSK by the client. This - * depends on session tickets being enabled (see - * #MBEDTLS_SSL_SESSION_TICKETS configuration option) though. - * If session tickets are disabled, a call to this function - * with a TLS 1.3 session, will not have any effect on the next - * handshake for the SSL context \p ssl. - * - * \param ssl The SSL context representing the connection which should - * be attempted to be setup using session resumption. This - * must be initialized via mbedtls_ssl_init() and bound to - * an SSL configuration via mbedtls_ssl_setup(), but - * the handshake must not yet have been started. - * \param session The session to be considered for session resumption. - * This must be a session previously exported via - * mbedtls_ssl_get_session(), and potentially serialized and - * deserialized through mbedtls_ssl_session_save() and - * mbedtls_ssl_session_load() in the meantime. - * - * \return \c 0 if successful. - * \return \c MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE if the session - * could not be loaded because one session has already been - * loaded. This error is non-fatal, and has no observable - * effect on the SSL context or the session that was attempted - * to be loaded. - * \return Another negative error code on other kinds of failure. - * - * \sa mbedtls_ssl_get_session() - * \sa mbedtls_ssl_session_load() - */ -int mbedtls_ssl_set_session(mbedtls_ssl_context *ssl, const mbedtls_ssl_session *session); -#endif /* MBEDTLS_SSL_CLI_C */ - -/** - * \brief Load serialized session data into a session structure. - * On client, this can be used for loading saved sessions - * before resuming them with mbedtls_ssl_set_session(). - * On server, this can be used for alternative implementations - * of session cache or session tickets. - * - * \warning If a peer certificate chain is associated with the session, - * the serialized state will only contain the peer's - * end-entity certificate and the result of the chain - * verification (unless verification was disabled), but not - * the rest of the chain. - * - * \see mbedtls_ssl_session_save() - * \see mbedtls_ssl_set_session() - * - * \param session The session structure to be populated. It must have been - * initialised with mbedtls_ssl_session_init() but not - * populated yet. - * \param buf The buffer holding the serialized session data. It must be a - * readable buffer of at least \p len bytes. - * \param len The size of the serialized data in bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_ALLOC_FAILED if memory allocation failed. - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if input data is invalid. - * \return #MBEDTLS_ERR_SSL_VERSION_MISMATCH if the serialized data - * was generated in a different version or configuration of - * Mbed TLS. - * \return Another negative value for other kinds of errors (for - * example, unsupported features in the embedded certificate). - */ -int mbedtls_ssl_session_load(mbedtls_ssl_session *session, - const unsigned char *buf, - size_t len); - -/** - * \brief Save session structure as serialized data in a buffer. - * On client, this can be used for saving session data, - * potentially in non-volatile storage, for resuming later. - * On server, this can be used for alternative implementations - * of session cache or session tickets. - * - * \see mbedtls_ssl_session_load() - * - * \param session The session structure to be saved. - * \param buf The buffer to write the serialized data to. It must be a - * writeable buffer of at least \p buf_len bytes, or may be \c - * NULL if \p buf_len is \c 0. - * \param buf_len The number of bytes available for writing in \p buf. - * \param olen The size in bytes of the data that has been or would have - * been written. It must point to a valid \c size_t. - * - * \note \p olen is updated to the correct value regardless of - * whether \p buf_len was large enough. This makes it possible - * to determine the necessary size by calling this function - * with \p buf set to \c NULL and \p buf_len to \c 0. - * - * \note For TLS 1.3 sessions, this feature is supported only if the - * MBEDTLS_SSL_SESSION_TICKETS configuration option is enabled, - * as in TLS 1.3 session resumption is possible only with - * tickets. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL if \p buf is too small. - * \return #MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE if the - * MBEDTLS_SSL_SESSION_TICKETS configuration option is disabled - * and the session is a TLS 1.3 session. - */ -int mbedtls_ssl_session_save(const mbedtls_ssl_session *session, - unsigned char *buf, - size_t buf_len, - size_t *olen); - -/** - * \brief Set the list of allowed ciphersuites and the preference - * order. First in the list has the highest preference. - * - * For TLS 1.2, the notion of ciphersuite determines both - * the key exchange mechanism and the suite of symmetric - * algorithms to be used during and after the handshake. - * - * For TLS 1.3 (in development), the notion of ciphersuite - * only determines the suite of symmetric algorithms to be - * used during and after the handshake, while key exchange - * mechanisms are configured separately. - * - * In Mbed TLS, ciphersuites for both TLS 1.2 and TLS 1.3 - * are configured via this function. For users of TLS 1.3, - * there will be separate API for the configuration of key - * exchange mechanisms. - * - * The list of ciphersuites passed to this function may - * contain a mixture of TLS 1.2 and TLS 1.3 ciphersuite - * identifiers. This is useful if negotiation of TLS 1.3 - * should be attempted, but a fallback to TLS 1.2 would - * be tolerated. - * - * \note By default, the server chooses its preferred - * ciphersuite among those that the client supports. If - * mbedtls_ssl_conf_preference_order() is called to prefer - * the client's preferences, the server instead chooses - * the client's preferred ciphersuite among those that - * the server supports. - * - * \warning The ciphersuites array \p ciphersuites is not copied. - * It must remain valid for the lifetime of the SSL - * configuration \p conf. - * - * \param conf The SSL configuration to modify. - * \param ciphersuites A 0-terminated list of IANA identifiers of supported - * ciphersuites, accessible through \c MBEDTLS_TLS_XXX - * and \c MBEDTLS_TLS1_3_XXX macros defined in - * ssl_ciphersuites.h. - */ -void mbedtls_ssl_conf_ciphersuites(mbedtls_ssl_config *conf, - const int *ciphersuites); - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) -/** - * \brief Set the supported key exchange modes for TLS 1.3 connections. - * - * In contrast to TLS 1.2, the ciphersuite concept in TLS 1.3 does not - * include the choice of key exchange mechanism. It is therefore not - * covered by the API mbedtls_ssl_conf_ciphersuites(). See the - * documentation of mbedtls_ssl_conf_ciphersuites() for more - * information on the ciphersuite concept in TLS 1.2 and TLS 1.3. - * - * The present function is specific to TLS 1.3 and allows users to - * configure the set of supported key exchange mechanisms in TLS 1.3. - * - * \param conf The SSL configuration the change should apply to. - * \param kex_modes A bitwise combination of one or more of the following: - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK - * This flag enables pure-PSK key exchanges. - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL - * This flag enables combined PSK-ephemeral key exchanges. - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL - * This flag enables pure-ephemeral key exchanges. - * For convenience, the following pre-defined macros are - * available for combinations of the above: - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_ALL - * Includes all of pure-PSK, PSK-ephemeral and pure-ephemeral. - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ALL - * Includes both pure-PSK and combined PSK-ephemeral - * key exchanges, but excludes pure-ephemeral key exchanges. - * - MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ALL - * Includes both pure-ephemeral and combined PSK-ephemeral - * key exchanges. - * - * \note If a PSK-based key exchange mode shall be supported, applications - * must also use the APIs mbedtls_ssl_conf_psk() or - * mbedtls_ssl_conf_psk_cb() or mbedtls_ssl_conf_psk_opaque() - * to configure the PSKs to be used. - * - * \note If a pure-ephemeral key exchange mode shall be supported, - * server-side applications must also provide a certificate via - * mbedtls_ssl_conf_own_cert(). - * - */ - -void mbedtls_ssl_conf_tls13_key_exchange_modes(mbedtls_ssl_config *conf, - const int kex_modes); -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 */ - -#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID) -#define MBEDTLS_SSL_UNEXPECTED_CID_IGNORE 0 -#define MBEDTLS_SSL_UNEXPECTED_CID_FAIL 1 -/** - * \brief Specify the length of Connection IDs for incoming - * encrypted DTLS records, as well as the behaviour - * on unexpected CIDs. - * - * By default, the CID length is set to \c 0, - * and unexpected CIDs are silently ignored. - * - * \param conf The SSL configuration to modify. - * \param len The length in Bytes of the CID fields in encrypted - * DTLS records using the CID mechanism. This must - * not be larger than #MBEDTLS_SSL_CID_OUT_LEN_MAX. - * \param ignore_other_cids This determines the stack's behaviour when - * receiving a record with an unexpected CID. - * Possible values are: - * - #MBEDTLS_SSL_UNEXPECTED_CID_IGNORE - * In this case, the record is silently ignored. - * - #MBEDTLS_SSL_UNEXPECTED_CID_FAIL - * In this case, the stack fails with the specific - * error code #MBEDTLS_ERR_SSL_UNEXPECTED_CID. - * - * \note The CID specification allows implementations to either - * use a common length for all incoming connection IDs or - * allow variable-length incoming IDs. Mbed TLS currently - * requires a common length for all connections sharing the - * same SSL configuration; this allows simpler parsing of - * record headers. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if \p len - * is too large. - */ -int mbedtls_ssl_conf_cid(mbedtls_ssl_config *conf, size_t len, - int ignore_other_cids); -#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Set the X.509 security profile used for verification - * - * \note The restrictions are enforced for all certificates in the - * chain. However, signatures in the handshake are not covered - * by this setting but by \b mbedtls_ssl_conf_sig_hashes(). - * - * \param conf SSL configuration - * \param profile Profile to use - */ -void mbedtls_ssl_conf_cert_profile(mbedtls_ssl_config *conf, - const mbedtls_x509_crt_profile *profile); - -/** - * \brief Set the data required to verify peer certificate - * - * \note See \c mbedtls_x509_crt_verify() for notes regarding the - * parameters ca_chain (maps to trust_ca for that function) - * and ca_crl. - * - * \param conf SSL configuration - * \param ca_chain trusted CA chain (meaning all fully trusted top-level CAs) - * \param ca_crl trusted CA CRLs - */ -void mbedtls_ssl_conf_ca_chain(mbedtls_ssl_config *conf, - mbedtls_x509_crt *ca_chain, - mbedtls_x509_crl *ca_crl); - -#if defined(MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED) -/** - * \brief Set DN hints sent to client in CertificateRequest message - * - * \note If not set, subject distinguished names (DNs) are taken - * from \c mbedtls_ssl_conf_ca_chain() - * or \c mbedtls_ssl_set_hs_ca_chain()) - * - * \param conf SSL configuration - * \param crt crt chain whose subject DNs are issuer DNs of client certs - * from which the client should select client peer certificate. - */ -static inline -void mbedtls_ssl_conf_dn_hints(mbedtls_ssl_config *conf, - const mbedtls_x509_crt *crt) -{ - conf->MBEDTLS_PRIVATE(dn_hints) = crt; -} -#endif /* MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED */ - -#if defined(MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK) -/** - * \brief Set the trusted certificate callback. - * - * This API allows to register the set of trusted certificates - * through a callback, instead of a linked list as configured - * by mbedtls_ssl_conf_ca_chain(). - * - * This is useful for example in contexts where a large number - * of CAs are used, and the inefficiency of maintaining them - * in a linked list cannot be tolerated. It is also useful when - * the set of trusted CAs needs to be modified frequently. - * - * See the documentation of `mbedtls_x509_crt_ca_cb_t` for - * more information. - * - * \param conf The SSL configuration to register the callback with. - * \param f_ca_cb The trusted certificate callback to use when verifying - * certificate chains. - * \param p_ca_cb The context to be passed to \p f_ca_cb (for example, - * a reference to a trusted CA database). - * - * \note This API is incompatible with mbedtls_ssl_conf_ca_chain(): - * Any call to this function overwrites the values set through - * earlier calls to mbedtls_ssl_conf_ca_chain() or - * mbedtls_ssl_conf_ca_cb(). - * - * \note This API is incompatible with CA indication in - * CertificateRequest messages: A server-side SSL context which - * is bound to an SSL configuration that uses a CA callback - * configured via mbedtls_ssl_conf_ca_cb(), and which requires - * client authentication, will send an empty CA list in the - * corresponding CertificateRequest message. - * - * \note This API is incompatible with mbedtls_ssl_set_hs_ca_chain(): - * If an SSL context is bound to an SSL configuration which uses - * CA callbacks configured via mbedtls_ssl_conf_ca_cb(), then - * calls to mbedtls_ssl_set_hs_ca_chain() have no effect. - * - * \note The use of this API disables the use of restartable ECC - * during X.509 CRT signature verification (but doesn't affect - * other uses). - * - * \warning This API is incompatible with the use of CRLs. Any call to - * mbedtls_ssl_conf_ca_cb() unsets CRLs configured through - * earlier calls to mbedtls_ssl_conf_ca_chain(). - * - * \warning In multi-threaded environments, the callback \p f_ca_cb - * must be thread-safe, and it is the user's responsibility - * to guarantee this (for example through a mutex - * contained in the callback context pointed to by \p p_ca_cb). - */ -void mbedtls_ssl_conf_ca_cb(mbedtls_ssl_config *conf, - mbedtls_x509_crt_ca_cb_t f_ca_cb, - void *p_ca_cb); -#endif /* MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK */ - -/** - * \brief Set own certificate chain and private key - * - * \note own_cert should contain in order from the bottom up your - * certificate chain. The top certificate (self-signed) - * can be omitted. - * - * \note On server, this function can be called multiple times to - * provision more than one cert/key pair (eg one ECDSA, one - * RSA with SHA-256, one RSA with SHA-1). An adequate - * certificate will be selected according to the client's - * advertised capabilities. In case multiple certificates are - * adequate, preference is given to the one set by the first - * call to this function, then second, etc. - * - * \note On client, only the first call has any effect. That is, - * only one client certificate can be provisioned. The - * server's preferences in its CertificateRequest message will - * be ignored and our only cert will be sent regardless of - * whether it matches those preferences - the server can then - * decide what it wants to do with it. - * - * \note The provided \p pk_key needs to match the public key in the - * first certificate in \p own_cert, or all handshakes using - * that certificate will fail. It is your responsibility - * to ensure that; this function will not perform any check. - * You may use mbedtls_pk_check_pair() in order to perform - * this check yourself, but be aware that this function can - * be computationally expensive on some key types. - * - * \param conf SSL configuration - * \param own_cert own public certificate chain - * \param pk_key own private key - * - * \return 0 on success or MBEDTLS_ERR_SSL_ALLOC_FAILED - */ -int mbedtls_ssl_conf_own_cert(mbedtls_ssl_config *conf, - mbedtls_x509_crt *own_cert, - mbedtls_pk_context *pk_key); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED) -/** - * \brief Configure pre-shared keys (PSKs) and their - * identities to be used in PSK-based ciphersuites. - * - * Only one PSK can be registered, through either - * mbedtls_ssl_conf_psk() or mbedtls_ssl_conf_psk_opaque(). - * If you attempt to register more than one PSK, this function - * fails, though this may change in future versions, which - * may add support for multiple PSKs. - * - * \note This is mainly useful for clients. Servers will usually - * want to use \c mbedtls_ssl_conf_psk_cb() instead. - * - * \note A PSK set by \c mbedtls_ssl_set_hs_psk() in the PSK callback - * takes precedence over a PSK configured by this function. - * - * \param conf The SSL configuration to register the PSK with. - * \param psk The pointer to the pre-shared key to use. - * \param psk_len The length of the pre-shared key in bytes. - * \param psk_identity The pointer to the pre-shared key identity. - * \param psk_identity_len The length of the pre-shared key identity - * in bytes. - * - * \note The PSK and its identity are copied internally and - * hence need not be preserved by the caller for the lifetime - * of the SSL configuration. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE if no more PSKs - * can be configured. In this case, the old PSK(s) remain intact. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ssl_conf_psk(mbedtls_ssl_config *conf, - const unsigned char *psk, size_t psk_len, - const unsigned char *psk_identity, size_t psk_identity_len); - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -/** - * \brief Configure one or more opaque pre-shared keys (PSKs) and - * their identities to be used in PSK-based ciphersuites. - * - * Only one PSK can be registered, through either - * mbedtls_ssl_conf_psk() or mbedtls_ssl_conf_psk_opaque(). - * If you attempt to register more than one PSK, this function - * fails, though this may change in future versions, which - * may add support for multiple PSKs. - * - * \note This is mainly useful for clients. Servers will usually - * want to use \c mbedtls_ssl_conf_psk_cb() instead. - * - * \note An opaque PSK set by \c mbedtls_ssl_set_hs_psk_opaque() in - * the PSK callback takes precedence over an opaque PSK - * configured by this function. - * - * \param conf The SSL configuration to register the PSK with. - * \param psk The identifier of the key slot holding the PSK. - * Until \p conf is destroyed or this function is successfully - * called again, the key slot \p psk must be populated with a - * key of type PSA_ALG_CATEGORY_KEY_DERIVATION whose policy - * allows its use for the key derivation algorithm applied - * in the handshake. - * \param psk_identity The pointer to the pre-shared key identity. - * \param psk_identity_len The length of the pre-shared key identity - * in bytes. - * - * \note The PSK identity hint is copied internally and hence need - * not be preserved by the caller for the lifetime of the - * SSL configuration. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE if no more PSKs - * can be configured. In this case, the old PSK(s) remain intact. - * \return Another negative error code on other kinds of failure. - */ -int mbedtls_ssl_conf_psk_opaque(mbedtls_ssl_config *conf, - mbedtls_svc_key_id_t psk, - const unsigned char *psk_identity, - size_t psk_identity_len); -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -/** - * \brief Set the pre-shared Key (PSK) for the current handshake. - * - * \note This should only be called inside the PSK callback, - * i.e. the function passed to \c mbedtls_ssl_conf_psk_cb(). - * - * \note A PSK set by this function takes precedence over a PSK - * configured by \c mbedtls_ssl_conf_psk(). - * - * \param ssl The SSL context to configure a PSK for. - * \param psk The pointer to the pre-shared key. - * \param psk_len The length of the pre-shared key in bytes. - * - * \return \c 0 if successful. - * \return An \c MBEDTLS_ERR_SSL_XXX error code on failure. - */ -int mbedtls_ssl_set_hs_psk(mbedtls_ssl_context *ssl, - const unsigned char *psk, size_t psk_len); - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -/** - * \brief Set an opaque pre-shared Key (PSK) for the current handshake. - * - * \note This should only be called inside the PSK callback, - * i.e. the function passed to \c mbedtls_ssl_conf_psk_cb(). - * - * \note An opaque PSK set by this function takes precedence over an - * opaque PSK configured by \c mbedtls_ssl_conf_psk_opaque(). - * - * \param ssl The SSL context to configure a PSK for. - * \param psk The identifier of the key slot holding the PSK. - * For the duration of the current handshake, the key slot - * must be populated with a key of type - * PSA_ALG_CATEGORY_KEY_DERIVATION whose policy allows its - * use for the key derivation algorithm - * applied in the handshake. - * - * \return \c 0 if successful. - * \return An \c MBEDTLS_ERR_SSL_XXX error code on failure. - */ -int mbedtls_ssl_set_hs_psk_opaque(mbedtls_ssl_context *ssl, - mbedtls_svc_key_id_t psk); -#endif /* MBEDTLS_USE_PSA_CRYPTO */ - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Set the PSK callback (server-side only). - * - * If set, the PSK callback is called for each - * handshake where a PSK-based ciphersuite was negotiated. - * The caller provides the identity received and wants to - * receive the actual PSK data and length. - * - * The callback has the following parameters: - * - \c void*: The opaque pointer \p p_psk. - * - \c mbedtls_ssl_context*: The SSL context to which - * the operation applies. - * - \c const unsigned char*: The PSK identity - * selected by the client. - * - \c size_t: The length of the PSK identity - * selected by the client. - * - * If a valid PSK identity is found, the callback should use - * \c mbedtls_ssl_set_hs_psk() or - * \c mbedtls_ssl_set_hs_psk_opaque() - * on the SSL context to set the correct PSK and return \c 0. - * Any other return value will result in a denied PSK identity. - * - * \note A dynamic PSK (i.e. set by the PSK callback) takes - * precedence over a static PSK (i.e. set by - * \c mbedtls_ssl_conf_psk() or - * \c mbedtls_ssl_conf_psk_opaque()). - * This means that if you set a PSK callback using this - * function, you don't need to set a PSK using - * \c mbedtls_ssl_conf_psk() or - * \c mbedtls_ssl_conf_psk_opaque()). - * - * \param conf The SSL configuration to register the callback with. - * \param f_psk The callback for selecting and setting the PSK based - * in the PSK identity chosen by the client. - * \param p_psk A pointer to an opaque structure to be passed to - * the callback, for example a PSK store. - */ -void mbedtls_ssl_conf_psk_cb(mbedtls_ssl_config *conf, - int (*f_psk)(void *, mbedtls_ssl_context *, const unsigned char *, - size_t), - void *p_psk); -#endif /* MBEDTLS_SSL_SRV_C */ -#endif /* MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED */ - -#if defined(MBEDTLS_DHM_C) && defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Set the Diffie-Hellman public P and G values - * from big-endian binary presentations. - * (Default values: MBEDTLS_DHM_RFC3526_MODP_2048_[PG]_BIN) - * - * \param conf SSL configuration - * \param dhm_P Diffie-Hellman-Merkle modulus in big-endian binary form - * \param P_len Length of DHM modulus - * \param dhm_G Diffie-Hellman-Merkle generator in big-endian binary form - * \param G_len Length of DHM generator - * - * \return 0 if successful - */ -int mbedtls_ssl_conf_dh_param_bin(mbedtls_ssl_config *conf, - const unsigned char *dhm_P, size_t P_len, - const unsigned char *dhm_G, size_t G_len); - -/** - * \brief Set the Diffie-Hellman public P and G values, - * read from existing context (server-side only) - * - * \param conf SSL configuration - * \param dhm_ctx Diffie-Hellman-Merkle context - * - * \return 0 if successful - */ -int mbedtls_ssl_conf_dh_param_ctx(mbedtls_ssl_config *conf, mbedtls_dhm_context *dhm_ctx); -#endif /* MBEDTLS_DHM_C && defined(MBEDTLS_SSL_SRV_C) */ - -#if defined(MBEDTLS_DHM_C) && defined(MBEDTLS_SSL_CLI_C) -/** - * \brief Set the minimum length for Diffie-Hellman parameters. - * (Client-side only.) - * (Default: 1024 bits.) - * - * \param conf SSL configuration - * \param bitlen Minimum bit length of the DHM prime - */ -void mbedtls_ssl_conf_dhm_min_bitlen(mbedtls_ssl_config *conf, - unsigned int bitlen); -#endif /* MBEDTLS_DHM_C && MBEDTLS_SSL_CLI_C */ - -#if defined(MBEDTLS_ECP_C) -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief Set the allowed curves in order of preference. - * - * On server: this only affects selection of the ECDHE curve; - * the curves used for ECDH and ECDSA are determined by the - * list of available certificates instead. - * - * On client: this affects the list of curves offered for any - * use. The server can override our preference order. - * - * Both sides: limits the set of curves accepted for use in - * ECDHE and in the peer's end-entity certificate. - * - * \deprecated Superseded by mbedtls_ssl_conf_groups(). - * - * \note This has no influence on which curves are allowed inside the - * certificate chains, see \c mbedtls_ssl_conf_cert_profile() - * for that. For the end-entity certificate however, the key - * will be accepted only if it is allowed both by this list - * and by the cert profile. - * - * \note This list should be ordered by decreasing preference - * (preferred curve first). - * - * \note The default list is the same set of curves that - * #mbedtls_x509_crt_profile_default allows, plus - * ECDHE-only curves selected according to the same criteria. - * The order favors curves with the lowest resource usage. - * - * \note New minor versions of Mbed TLS may extend this list, - * for example if new curves are added to the library. - * New minor versions of Mbed TLS will not remove items - * from this list unless serious security concerns require it. - * New minor versions of Mbed TLS may change the order in - * keeping with the general principle of favoring the lowest - * resource usage. - * - * \param conf SSL configuration - * \param curves Ordered list of allowed curves, - * terminated by MBEDTLS_ECP_DP_NONE. - */ -void MBEDTLS_DEPRECATED mbedtls_ssl_conf_curves(mbedtls_ssl_config *conf, - const mbedtls_ecp_group_id *curves); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ -#endif /* MBEDTLS_ECP_C */ - -/** - * \brief Set the allowed groups in order of preference. - * - * On server: This only affects the choice of key agreement mechanism - * - * On client: this affects the list of groups offered for any - * use. The server can override our preference order. - * - * Both sides: limits the set of groups accepted for use in - * key sharing. - * - * \note This function replaces the deprecated mbedtls_ssl_conf_curves(), - * which only allows ECP curves to be configured. - * - * \note The most recent invocation of either mbedtls_ssl_conf_curves() - * or mbedtls_ssl_conf_groups() nullifies all previous invocations - * of both. - * - * \note This list should be ordered by decreasing preference - * (preferred group first). - * - * \note When this function is not called, a default list is used, - * consisting of all supported curves at 255 bits and above, - * and all supported finite fields at 2048 bits and above. - * The order favors groups with the lowest resource usage. - * - * \note New minor versions of Mbed TLS will not remove items - * from the default list unless serious security concerns require it. - * New minor versions of Mbed TLS may change the order in - * keeping with the general principle of favoring the lowest - * resource usage. - * - * \param conf SSL configuration - * \param groups List of allowed groups ordered by preference, terminated by 0. - * Must contain valid IANA NamedGroup IDs (provided via either an integer - * or using MBEDTLS_TLS1_3_NAMED_GROUP_XXX macros). - */ -void mbedtls_ssl_conf_groups(mbedtls_ssl_config *conf, - const uint16_t *groups); - -#if defined(MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED) -#if !defined(MBEDTLS_DEPRECATED_REMOVED) && defined(MBEDTLS_SSL_PROTO_TLS1_2) -/** - * \brief Set the allowed hashes for signatures during the handshake. - * - * \note This only affects which hashes are offered and can be used - * for signatures during the handshake. Hashes for message - * authentication and the TLS PRF are controlled by the - * ciphersuite, see \c mbedtls_ssl_conf_ciphersuites(). Hashes - * used for certificate signature are controlled by the - * verification profile, see \c mbedtls_ssl_conf_cert_profile(). - * - * \deprecated Superseded by mbedtls_ssl_conf_sig_algs(). - * - * \note This list should be ordered by decreasing preference - * (preferred hash first). - * - * \note By default, all supported hashes whose length is at least - * 256 bits are allowed. This is the same set as the default - * for certificate verification - * (#mbedtls_x509_crt_profile_default). - * The preference order is currently unspecified and may - * change in future versions. - * - * \note New minor versions of Mbed TLS may extend this list, - * for example if new curves are added to the library. - * New minor versions of Mbed TLS will not remove items - * from this list unless serious security concerns require it. - * - * \param conf SSL configuration - * \param hashes Ordered list of allowed signature hashes, - * terminated by \c MBEDTLS_MD_NONE. - */ -void MBEDTLS_DEPRECATED mbedtls_ssl_conf_sig_hashes(mbedtls_ssl_config *conf, - const int *hashes); -#endif /* !MBEDTLS_DEPRECATED_REMOVED && MBEDTLS_SSL_PROTO_TLS1_2 */ - -/** - * \brief Configure allowed signature algorithms for use in TLS - * - * \param conf The SSL configuration to use. - * \param sig_algs List of allowed IANA values for TLS 1.3 signature algorithms, - * terminated by #MBEDTLS_TLS1_3_SIG_NONE. The list must remain - * available throughout the lifetime of the conf object. - * - For TLS 1.3, values of \c MBEDTLS_TLS1_3_SIG_XXXX should be - * used. - * - For TLS 1.2, values should be given as - * "(HashAlgorithm << 8) | SignatureAlgorithm". - */ -void mbedtls_ssl_conf_sig_algs(mbedtls_ssl_config *conf, - const uint16_t *sig_algs); -#endif /* MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Set or reset the hostname to check against the received - * peer certificate. On a client, this also sets the - * ServerName TLS extension, if that extension is enabled. - * On a TLS 1.3 client, this also sets the server name in - * the session resumption ticket, if that feature is enabled. - * - * \param ssl SSL context - * \param hostname The server hostname. This may be \c NULL to clear - * the hostname. - * - * \note Maximum hostname length #MBEDTLS_SSL_MAX_HOST_NAME_LEN. - * - * \note If the hostname is \c NULL on a client, then the server - * is not authenticated: it only needs to have a valid - * certificate, not a certificate matching its name. - * Therefore you should always call this function on a client, - * unless the connection is set up to only allow - * pre-shared keys, or in scenarios where server - * impersonation is not a concern. See the documentation of - * #MBEDTLS_ERR_SSL_CERTIFICATE_VERIFICATION_WITHOUT_HOSTNAME - * for more details. - * - * \return 0 if successful, #MBEDTLS_ERR_SSL_ALLOC_FAILED on - * allocation failure, #MBEDTLS_ERR_SSL_BAD_INPUT_DATA on - * too long input hostname. - * - * Hostname set to the one provided on success (cleared - * when NULL). On allocation failure hostname is cleared. - * On too long input failure, old hostname is unchanged. - */ -int mbedtls_ssl_set_hostname(mbedtls_ssl_context *ssl, const char *hostname); - -/** - * \brief Get the hostname that checked against the received - * server certificate. It is used to set the ServerName - * TLS extension, too, if that extension is enabled. - * (client-side only) - * - * \param ssl SSL context - * - * \return const pointer to the hostname value - */ -static inline const char *mbedtls_ssl_get_hostname(mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(hostname); -} -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_SSL_SERVER_NAME_INDICATION) -/** - * \brief Retrieve SNI extension value for the current handshake. - * Available in \c f_cert_cb of \c mbedtls_ssl_conf_cert_cb(), - * this is the same value passed to \c f_sni callback of - * \c mbedtls_ssl_conf_sni() and may be used instead of - * \c mbedtls_ssl_conf_sni(). - * - * \param ssl SSL context - * \param name_len pointer into which to store length of returned value. - * 0 if SNI extension is not present or not yet processed. - * - * \return const pointer to SNI extension value. - * - value is valid only when called in \c f_cert_cb - * registered with \c mbedtls_ssl_conf_cert_cb(). - * - value is NULL if SNI extension is not present. - * - value is not '\0'-terminated. Use \c name_len for len. - * - value must not be freed. - */ -const unsigned char *mbedtls_ssl_get_hs_sni(mbedtls_ssl_context *ssl, - size_t *name_len); - -/** - * \brief Set own certificate and key for the current handshake - * - * \note Same as \c mbedtls_ssl_conf_own_cert() but for use within - * the SNI callback or the certificate selection callback. - * - * \note Passing null \c own_cert clears the certificate list for - * the current handshake. - * - * \param ssl SSL context - * \param own_cert own public certificate chain - * \param pk_key own private key - * - * \return 0 on success or MBEDTLS_ERR_SSL_ALLOC_FAILED - */ -int mbedtls_ssl_set_hs_own_cert(mbedtls_ssl_context *ssl, - mbedtls_x509_crt *own_cert, - mbedtls_pk_context *pk_key); - -/** - * \brief Set the data required to verify peer certificate for the - * current handshake - * - * \note Same as \c mbedtls_ssl_conf_ca_chain() but for use within - * the SNI callback or the certificate selection callback. - * - * \param ssl SSL context - * \param ca_chain trusted CA chain (meaning all fully trusted top-level CAs) - * \param ca_crl trusted CA CRLs - */ -void mbedtls_ssl_set_hs_ca_chain(mbedtls_ssl_context *ssl, - mbedtls_x509_crt *ca_chain, - mbedtls_x509_crl *ca_crl); - -#if defined(MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED) -/** - * \brief Set DN hints sent to client in CertificateRequest message - * - * \note Same as \c mbedtls_ssl_conf_dn_hints() but for use within - * the SNI callback or the certificate selection callback. - * - * \param ssl SSL context - * \param crt crt chain whose subject DNs are issuer DNs of client certs - * from which the client should select client peer certificate. - */ -void mbedtls_ssl_set_hs_dn_hints(mbedtls_ssl_context *ssl, - const mbedtls_x509_crt *crt); -#endif /* MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED */ - -/** - * \brief Set authmode for the current handshake. - * - * \note Same as \c mbedtls_ssl_conf_authmode() but for use within - * the SNI callback or the certificate selection callback. - * - * \param ssl SSL context - * \param authmode MBEDTLS_SSL_VERIFY_NONE, MBEDTLS_SSL_VERIFY_OPTIONAL or - * MBEDTLS_SSL_VERIFY_REQUIRED - */ -void mbedtls_ssl_set_hs_authmode(mbedtls_ssl_context *ssl, - int authmode); - -/** - * \brief Set server side ServerName TLS extension callback - * (optional, server-side only). - * - * If set, the ServerName callback is called whenever the - * server receives a ServerName TLS extension from the client - * during a handshake. The ServerName callback has the - * following parameters: (void *parameter, mbedtls_ssl_context *ssl, - * const unsigned char *hostname, size_t len). If a suitable - * certificate is found, the callback must set the - * certificate(s) and key(s) to use with \c - * mbedtls_ssl_set_hs_own_cert() (can be called repeatedly), - * and may optionally adjust the CA and associated CRL with \c - * mbedtls_ssl_set_hs_ca_chain() as well as the client - * authentication mode with \c mbedtls_ssl_set_hs_authmode(), - * then must return 0. If no matching name is found, the - * callback may return non-zero to abort the handshake. - * - * \param conf SSL configuration - * \param f_sni verification function - * \param p_sni verification parameter - */ -void mbedtls_ssl_conf_sni(mbedtls_ssl_config *conf, - int (*f_sni)(void *, mbedtls_ssl_context *, const unsigned char *, - size_t), - void *p_sni); -#endif /* MBEDTLS_SSL_SERVER_NAME_INDICATION */ - -#if defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) -/** - * \brief Set the EC J-PAKE password for current handshake. - * - * \note An internal copy is made, and destroyed as soon as the - * handshake is completed, or when the SSL context is reset or - * freed. - * - * \note The SSL context needs to be already set up. The right place - * to call this function is between \c mbedtls_ssl_setup() or - * \c mbedtls_ssl_reset() and \c mbedtls_ssl_handshake(). - * Password cannot be empty (see RFC 8236). - * - * \param ssl SSL context - * \param pw EC J-PAKE password (pre-shared secret). It cannot be empty - * \param pw_len length of pw in bytes - * - * \return 0 on success, or a negative error code. - */ -int mbedtls_ssl_set_hs_ecjpake_password(mbedtls_ssl_context *ssl, - const unsigned char *pw, - size_t pw_len); - -/** - * \brief Set the EC J-PAKE opaque password for current handshake. - * - * \note The key must remain valid until the handshake is over. - * - * \note The SSL context needs to be already set up. The right place - * to call this function is between \c mbedtls_ssl_setup() or - * \c mbedtls_ssl_reset() and \c mbedtls_ssl_handshake(). - * - * \param ssl SSL context - * \param pwd EC J-PAKE opaque password - * - * \return 0 on success, or a negative error code. - */ -int mbedtls_ssl_set_hs_ecjpake_password_opaque(mbedtls_ssl_context *ssl, - mbedtls_svc_key_id_t pwd); -#endif /*MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED */ - -#if defined(MBEDTLS_SSL_ALPN) -/** - * \brief Set the supported Application Layer Protocols. - * - * \param conf SSL configuration - * \param protos Pointer to a NULL-terminated list of supported protocols, - * in decreasing preference order. The pointer to the list is - * recorded by the library for later reference as required, so - * the lifetime of the table must be at least as long as the - * lifetime of the SSL configuration structure. - * - * \return 0 on success, or MBEDTLS_ERR_SSL_BAD_INPUT_DATA. - */ -int mbedtls_ssl_conf_alpn_protocols(mbedtls_ssl_config *conf, const char **protos); - -/** - * \brief Get the name of the negotiated Application Layer Protocol. - * This function should be called after the handshake is - * completed. - * - * \param ssl SSL context - * - * \return Protocol name, or NULL if no protocol was negotiated. - */ -const char *mbedtls_ssl_get_alpn_protocol(const mbedtls_ssl_context *ssl); -#endif /* MBEDTLS_SSL_ALPN */ - -#if defined(MBEDTLS_SSL_DTLS_SRTP) -#if defined(MBEDTLS_DEBUG_C) -static inline const char *mbedtls_ssl_get_srtp_profile_as_string(mbedtls_ssl_srtp_profile profile) -{ - switch (profile) { - case MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_80: - return "MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_80"; - case MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_32: - return "MBEDTLS_TLS_SRTP_AES128_CM_HMAC_SHA1_32"; - case MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_80: - return "MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_80"; - case MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_32: - return "MBEDTLS_TLS_SRTP_NULL_HMAC_SHA1_32"; - default: break; - } - return ""; -} -#endif /* MBEDTLS_DEBUG_C */ -/** - * \brief Manage support for mki(master key id) value - * in use_srtp extension. - * MKI is an optional part of SRTP used for key management - * and re-keying. See RFC3711 section 3.1 for details. - * The default value is - * #MBEDTLS_SSL_DTLS_SRTP_MKI_UNSUPPORTED. - * - * \param conf The SSL configuration to manage mki support. - * \param support_mki_value Enable or disable mki usage. Values are - * #MBEDTLS_SSL_DTLS_SRTP_MKI_UNSUPPORTED - * or #MBEDTLS_SSL_DTLS_SRTP_MKI_SUPPORTED. - */ -void mbedtls_ssl_conf_srtp_mki_value_supported(mbedtls_ssl_config *conf, - int support_mki_value); - -/** - * \brief Set the supported DTLS-SRTP protection profiles. - * - * \param conf SSL configuration - * \param profiles Pointer to a List of MBEDTLS_TLS_SRTP_UNSET terminated - * supported protection profiles - * in decreasing preference order. - * The pointer to the list is recorded by the library - * for later reference as required, so the lifetime - * of the table must be at least as long as the lifetime - * of the SSL configuration structure. - * The list must not hold more than - * MBEDTLS_TLS_SRTP_MAX_PROFILE_LIST_LENGTH elements - * (excluding the terminating MBEDTLS_TLS_SRTP_UNSET). - * - * \return 0 on success - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA when the list of - * protection profiles is incorrect. - */ -int mbedtls_ssl_conf_dtls_srtp_protection_profiles - (mbedtls_ssl_config *conf, - const mbedtls_ssl_srtp_profile *profiles); - -/** - * \brief Set the mki_value for the current DTLS-SRTP session. - * - * \param ssl SSL context to use. - * \param mki_value The MKI value to set. - * \param mki_len The length of the MKI value. - * - * \note This function is relevant on client side only. - * The server discovers the mki value during handshake. - * A mki value set on server side using this function - * is ignored. - * - * \return 0 on success - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA - * \return #MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE - */ -int mbedtls_ssl_dtls_srtp_set_mki_value(mbedtls_ssl_context *ssl, - unsigned char *mki_value, - uint16_t mki_len); -/** - * \brief Get the negotiated DTLS-SRTP information: - * Protection profile and MKI value. - * - * \warning This function must be called after the handshake is - * completed. The value returned by this function must - * not be trusted or acted upon before the handshake completes. - * - * \param ssl The SSL context to query. - * \param dtls_srtp_info The negotiated DTLS-SRTP information: - * - Protection profile in use. - * A direct mapping of the iana defined value for protection - * profile on an uint16_t. - http://www.iana.org/assignments/srtp-protection/srtp-protection.xhtml - * #MBEDTLS_TLS_SRTP_UNSET if the use of SRTP was not negotiated - * or peer's Hello packet was not parsed yet. - * - mki size and value( if size is > 0 ). - */ -void mbedtls_ssl_get_dtls_srtp_negotiation_result(const mbedtls_ssl_context *ssl, - mbedtls_dtls_srtp_info *dtls_srtp_info); -#endif /* MBEDTLS_SSL_DTLS_SRTP */ - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief Set the maximum supported version sent from the client side - * and/or accepted at the server side. - * - * See also the documentation of mbedtls_ssl_conf_min_version(). - * - * \note This ignores ciphersuites from higher versions. - * - * \note This function is deprecated and has been replaced by - * \c mbedtls_ssl_conf_max_tls_version(). - * - * \param conf SSL configuration - * \param major Major version number (#MBEDTLS_SSL_MAJOR_VERSION_3) - * \param minor Minor version number - * (#MBEDTLS_SSL_MINOR_VERSION_3 for (D)TLS 1.2, - * #MBEDTLS_SSL_MINOR_VERSION_4 for TLS 1.3) - */ -void MBEDTLS_DEPRECATED mbedtls_ssl_conf_max_version(mbedtls_ssl_config *conf, int major, - int minor); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/** - * \brief Set the maximum supported version sent from the client side - * and/or accepted at the server side. - * - * \note After the handshake, you can call - * mbedtls_ssl_get_version_number() to see what version was - * negotiated. - * - * \param conf SSL configuration - * \param tls_version TLS protocol version number (\c mbedtls_ssl_protocol_version) - * (#MBEDTLS_SSL_VERSION_UNKNOWN is not valid) - */ -static inline void mbedtls_ssl_conf_max_tls_version(mbedtls_ssl_config *conf, - mbedtls_ssl_protocol_version tls_version) -{ - conf->MBEDTLS_PRIVATE(max_tls_version) = tls_version; -} - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief Set the minimum accepted SSL/TLS protocol version - * - * \note By default, all supported versions are accepted. - * Future versions of the library may disable older - * protocol versions by default if they become deprecated. - * - * \note The following versions are supported (if enabled at - * compile time): - * - (D)TLS 1.2: \p major = #MBEDTLS_SSL_MAJOR_VERSION_3, - * \p minor = #MBEDTLS_SSL_MINOR_VERSION_3 - * - TLS 1.3: \p major = #MBEDTLS_SSL_MAJOR_VERSION_3, - * \p minor = #MBEDTLS_SSL_MINOR_VERSION_4 - * - * Note that the numbers in the constant names are the - * TLS internal protocol numbers, and the minor versions - * differ by one from the human-readable versions! - * - * \note Input outside of the SSL_MAX_XXXXX_VERSION and - * SSL_MIN_XXXXX_VERSION range is ignored. - * - * \note After the handshake, you can call - * mbedtls_ssl_get_version_number() to see what version was - * negotiated. - * - * \note This function is deprecated and has been replaced by - * \c mbedtls_ssl_conf_min_tls_version(). - * - * \param conf SSL configuration - * \param major Major version number (#MBEDTLS_SSL_MAJOR_VERSION_3) - * \param minor Minor version number - * (#MBEDTLS_SSL_MINOR_VERSION_3 for (D)TLS 1.2, - * #MBEDTLS_SSL_MINOR_VERSION_4 for TLS 1.3) - */ -void MBEDTLS_DEPRECATED mbedtls_ssl_conf_min_version(mbedtls_ssl_config *conf, int major, - int minor); -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/** - * \brief Set the minimum supported version sent from the client side - * and/or accepted at the server side. - * - * \note After the handshake, you can call - * mbedtls_ssl_get_version_number() to see what version was - * negotiated. - * - * \param conf SSL configuration - * \param tls_version TLS protocol version number (\c mbedtls_ssl_protocol_version) - * (#MBEDTLS_SSL_VERSION_UNKNOWN is not valid) - */ -static inline void mbedtls_ssl_conf_min_tls_version(mbedtls_ssl_config *conf, - mbedtls_ssl_protocol_version tls_version) -{ - conf->MBEDTLS_PRIVATE(min_tls_version) = tls_version; -} - -#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC) -/** - * \brief Enable or disable Encrypt-then-MAC - * (Default: MBEDTLS_SSL_ETM_ENABLED) - * - * \note This should always be enabled, it is a security - * improvement, and should not cause any interoperability - * issue (used only if the peer supports it too). - * - * \param conf SSL configuration - * \param etm MBEDTLS_SSL_ETM_ENABLED or MBEDTLS_SSL_ETM_DISABLED - */ -void mbedtls_ssl_conf_encrypt_then_mac(mbedtls_ssl_config *conf, char etm); -#endif /* MBEDTLS_SSL_ENCRYPT_THEN_MAC */ - -#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET) -/** - * \brief Enable or disable Extended Master Secret negotiation. - * (Default: MBEDTLS_SSL_EXTENDED_MS_ENABLED) - * - * \note This should always be enabled, it is a security fix to the - * protocol, and should not cause any interoperability issue - * (used only if the peer supports it too). - * - * \param conf SSL configuration - * \param ems MBEDTLS_SSL_EXTENDED_MS_ENABLED or MBEDTLS_SSL_EXTENDED_MS_DISABLED - */ -void mbedtls_ssl_conf_extended_master_secret(mbedtls_ssl_config *conf, char ems); -#endif /* MBEDTLS_SSL_EXTENDED_MASTER_SECRET */ - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Whether to send a list of acceptable CAs in - * CertificateRequest messages. - * (Default: do send) - * - * \param conf SSL configuration - * \param cert_req_ca_list MBEDTLS_SSL_CERT_REQ_CA_LIST_ENABLED or - * MBEDTLS_SSL_CERT_REQ_CA_LIST_DISABLED - */ -void mbedtls_ssl_conf_cert_req_ca_list(mbedtls_ssl_config *conf, - char cert_req_ca_list); -#endif /* MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH) -/** - * \brief Set the maximum fragment length to emit and/or negotiate. - * (Typical: the smaller of #MBEDTLS_SSL_IN_CONTENT_LEN and - * #MBEDTLS_SSL_OUT_CONTENT_LEN, usually `2^14` bytes) - * (Server: set maximum fragment length to emit, - * usually negotiated by the client during handshake) - * (Client: set maximum fragment length to emit *and* - * negotiate with the server during handshake) - * (Default: #MBEDTLS_SSL_MAX_FRAG_LEN_NONE) - * - * \note On the client side, the maximum fragment length extension - * *will not* be used, unless the maximum fragment length has - * been set via this function to a value different than - * #MBEDTLS_SSL_MAX_FRAG_LEN_NONE. - * - * \note With TLS, this currently only affects ApplicationData (sent - * with \c mbedtls_ssl_read()), not handshake messages. - * With DTLS, this affects both ApplicationData and handshake. - * - * \note Defragmentation of TLS handshake messages is supported - * with some limitations. See the documentation of - * mbedtls_ssl_handshake() for details. - * - * \note This sets the maximum length for a record's payload, - * excluding record overhead that will be added to it, see - * \c mbedtls_ssl_get_record_expansion(). - * - * \note For DTLS, it is also possible to set a limit for the total - * size of datagrams passed to the transport layer, including - * record overhead, see \c mbedtls_ssl_set_mtu(). - * - * \param conf SSL configuration - * \param mfl_code Code for maximum fragment length (allowed values: - * MBEDTLS_SSL_MAX_FRAG_LEN_512, MBEDTLS_SSL_MAX_FRAG_LEN_1024, - * MBEDTLS_SSL_MAX_FRAG_LEN_2048, MBEDTLS_SSL_MAX_FRAG_LEN_4096) - * - * \return 0 if successful or MBEDTLS_ERR_SSL_BAD_INPUT_DATA - */ -int mbedtls_ssl_conf_max_frag_len(mbedtls_ssl_config *conf, unsigned char mfl_code); -#endif /* MBEDTLS_SSL_MAX_FRAGMENT_LENGTH */ - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Pick the ciphersuites order according to the second parameter - * in the SSL Server module (MBEDTLS_SSL_SRV_C). - * (Default, if never called: MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_SERVER) - * - * \param conf SSL configuration - * \param order Server or client (MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_SERVER - * or MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_CLIENT) - */ -void mbedtls_ssl_conf_preference_order(mbedtls_ssl_config *conf, int order); -#endif /* MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && defined(MBEDTLS_SSL_CLI_C) -/** - * \brief Enable / Disable TLS 1.2 session tickets (client only, - * TLS 1.2 only). Enabled by default. - * - * \note On server, use \c mbedtls_ssl_conf_session_tickets_cb(). - * - * \param conf SSL configuration - * \param use_tickets Enable or disable (#MBEDTLS_SSL_SESSION_TICKETS_ENABLED or - * #MBEDTLS_SSL_SESSION_TICKETS_DISABLED) - */ -void mbedtls_ssl_conf_session_tickets(mbedtls_ssl_config *conf, int use_tickets); - -#if defined(MBEDTLS_SSL_PROTO_TLS1_3) -/** - * \brief Enable / Disable handling of TLS 1.3 NewSessionTicket messages - * (client only, TLS 1.3 only). - * - * The handling of TLS 1.3 NewSessionTicket messages is disabled by - * default. - * - * In TLS 1.3, servers may send a NewSessionTicket message at any time, - * and may send multiple NewSessionTicket messages. By default, TLS 1.3 - * clients ignore NewSessionTicket messages. - * - * To support session tickets in TLS 1.3 clients, call this function - * with #MBEDTLS_SSL_TLS1_3_SIGNAL_NEW_SESSION_TICKETS_ENABLED. When - * this is enabled, when a client receives a NewSessionTicket message, - * the next call to a message processing functions (notably - * mbedtls_ssl_handshake() and mbedtls_ssl_read()) will return - * #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET. The client should then - * call mbedtls_ssl_get_session() to retrieve the session ticket before - * calling the same message processing function again. - * - * \param conf SSL configuration - * \param signal_new_session_tickets Enable or disable - * (#MBEDTLS_SSL_TLS1_3_SIGNAL_NEW_SESSION_TICKETS_ENABLED or - * #MBEDTLS_SSL_TLS1_3_SIGNAL_NEW_SESSION_TICKETS_DISABLED) - */ -void mbedtls_ssl_conf_tls13_enable_signal_new_session_tickets( - mbedtls_ssl_config *conf, int signal_new_session_tickets); - -#endif /* MBEDTLS_SSL_PROTO_TLS1_3 */ -#endif /* MBEDTLS_SSL_SESSION_TICKETS && MBEDTLS_SSL_CLI_C */ - -#if defined(MBEDTLS_SSL_SESSION_TICKETS) && \ - defined(MBEDTLS_SSL_SRV_C) && \ - defined(MBEDTLS_SSL_PROTO_TLS1_3) -/** - * \brief Number of NewSessionTicket messages for the server to send - * after handshake completion. - * - * \note The default value is - * \c MBEDTLS_SSL_TLS1_3_DEFAULT_NEW_SESSION_TICKETS. - * - * \note In case of a session resumption, this setting only partially apply. - * At most one ticket is sent in that case to just renew the pool of - * tickets of the client. The rationale is to avoid the number of - * tickets on the server to become rapidly out of control when the - * server has the same configuration for all its connection instances. - * - * \param conf SSL configuration - * \param num_tickets Number of NewSessionTicket. - * - */ -void mbedtls_ssl_conf_new_session_tickets(mbedtls_ssl_config *conf, - uint16_t num_tickets); -#endif /* MBEDTLS_SSL_SESSION_TICKETS && - MBEDTLS_SSL_SRV_C && - MBEDTLS_SSL_PROTO_TLS1_3*/ - -#if defined(MBEDTLS_SSL_RENEGOTIATION) -/** - * \brief Enable / Disable renegotiation support for connection when - * initiated by peer - * (Default: MBEDTLS_SSL_RENEGOTIATION_DISABLED) - * - * \warning It is recommended to always disable renegotiation unless you - * know you need it and you know what you're doing. In the - * past, there have been several issues associated with - * renegotiation or a poor understanding of its properties. - * - * \note Server-side, enabling renegotiation also makes the server - * susceptible to a resource DoS by a malicious client. - * - * \param conf SSL configuration - * \param renegotiation Enable or disable (MBEDTLS_SSL_RENEGOTIATION_ENABLED or - * MBEDTLS_SSL_RENEGOTIATION_DISABLED) - */ -void mbedtls_ssl_conf_renegotiation(mbedtls_ssl_config *conf, int renegotiation); -#endif /* MBEDTLS_SSL_RENEGOTIATION */ - -/** - * \brief Prevent or allow legacy renegotiation. - * (Default: MBEDTLS_SSL_LEGACY_NO_RENEGOTIATION) - * - * MBEDTLS_SSL_LEGACY_NO_RENEGOTIATION allows connections to - * be established even if the peer does not support - * secure renegotiation, but does not allow renegotiation - * to take place if not secure. - * (Interoperable and secure option) - * - * MBEDTLS_SSL_LEGACY_ALLOW_RENEGOTIATION allows renegotiations - * with non-upgraded peers. Allowing legacy renegotiation - * makes the connection vulnerable to specific man in the - * middle attacks. (See RFC 5746) - * (Most interoperable and least secure option) - * - * MBEDTLS_SSL_LEGACY_BREAK_HANDSHAKE breaks off connections - * if peer does not support secure renegotiation. Results - * in interoperability issues with non-upgraded peers - * that do not support renegotiation altogether. - * (Most secure option, interoperability issues) - * - * \param conf SSL configuration - * \param allow_legacy Prevent or allow (SSL_NO_LEGACY_RENEGOTIATION, - * SSL_ALLOW_LEGACY_RENEGOTIATION or - * MBEDTLS_SSL_LEGACY_BREAK_HANDSHAKE) - */ -void mbedtls_ssl_conf_legacy_renegotiation(mbedtls_ssl_config *conf, int allow_legacy); - -#if defined(MBEDTLS_SSL_RENEGOTIATION) -/** - * \brief Enforce renegotiation requests. - * (Default: enforced, max_records = 16) - * - * When we request a renegotiation, the peer can comply or - * ignore the request. This function allows us to decide - * whether to enforce our renegotiation requests by closing - * the connection if the peer doesn't comply. - * - * However, records could already be in transit from the peer - * when the request is emitted. In order to increase - * reliability, we can accept a number of records before the - * expected handshake records. - * - * The optimal value is highly dependent on the specific usage - * scenario. - * - * \note With DTLS and server-initiated renegotiation, the - * HelloRequest is retransmitted every time mbedtls_ssl_read() times - * out or receives Application Data, until: - * - max_records records have beens seen, if it is >= 0, or - * - the number of retransmits that would happen during an - * actual handshake has been reached. - * Please remember the request might be lost a few times - * if you consider setting max_records to a really low value. - * - * \warning On client, the grace period can only happen during - * mbedtls_ssl_read(), as opposed to mbedtls_ssl_write() and mbedtls_ssl_renegotiate() - * which always behave as if max_record was 0. The reason is, - * if we receive application data from the server, we need a - * place to write it, which only happens during mbedtls_ssl_read(). - * - * \param conf SSL configuration - * \param max_records Use MBEDTLS_SSL_RENEGOTIATION_NOT_ENFORCED if you don't want to - * enforce renegotiation, or a non-negative value to enforce - * it but allow for a grace period of max_records records. - */ -void mbedtls_ssl_conf_renegotiation_enforced(mbedtls_ssl_config *conf, int max_records); - -/** - * \brief Set record counter threshold for periodic renegotiation. - * (Default: 2^48 - 1) - * - * Renegotiation is automatically triggered when a record - * counter (outgoing or incoming) crosses the defined - * threshold. The default value is meant to prevent the - * connection from being closed when the counter is about to - * reached its maximal value (it is not allowed to wrap). - * - * Lower values can be used to enforce policies such as "keys - * must be refreshed every N packets with cipher X". - * - * The renegotiation period can be disabled by setting - * conf->disable_renegotiation to - * MBEDTLS_SSL_RENEGOTIATION_DISABLED. - * - * \note When the configured transport is - * MBEDTLS_SSL_TRANSPORT_DATAGRAM the maximum renegotiation - * period is 2^48 - 1, and for MBEDTLS_SSL_TRANSPORT_STREAM, - * the maximum renegotiation period is 2^64 - 1. - * - * \param conf SSL configuration - * \param period The threshold value: a big-endian 64-bit number. - */ -void mbedtls_ssl_conf_renegotiation_period(mbedtls_ssl_config *conf, - const unsigned char period[8]); -#endif /* MBEDTLS_SSL_RENEGOTIATION */ - -/** - * \brief Check if there is data already read from the - * underlying transport but not yet processed. - * - * \param ssl SSL context - * - * \return 0 if nothing's pending, 1 otherwise. - * - * \note This is different in purpose and behaviour from - * \c mbedtls_ssl_get_bytes_avail in that it considers - * any kind of unprocessed data, not only unread - * application data. If \c mbedtls_ssl_get_bytes - * returns a non-zero value, this function will - * also signal pending data, but the converse does - * not hold. For example, in DTLS there might be - * further records waiting to be processed from - * the current underlying transport's datagram. - * - * \note If this function returns 1 (data pending), this - * does not imply that a subsequent call to - * \c mbedtls_ssl_read will provide any data; - * e.g., the unprocessed data might turn out - * to be an alert or a handshake message. - * - * \note This function is useful in the following situation: - * If the SSL/TLS module successfully returns from an - * operation - e.g. a handshake or an application record - * read - and you're awaiting incoming data next, you - * must not immediately idle on the underlying transport - * to have data ready, but you need to check the value - * of this function first. The reason is that the desired - * data might already be read but not yet processed. - * If, in contrast, a previous call to the SSL/TLS module - * returned MBEDTLS_ERR_SSL_WANT_READ, it is not necessary - * to call this function, as the latter error code entails - * that all internal data has been processed. - * - */ -int mbedtls_ssl_check_pending(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the number of application data bytes - * remaining to be read from the current record. - * - * \param ssl SSL context - * - * \return How many bytes are available in the application - * data record read buffer. - * - * \note When working over a datagram transport, this is - * useful to detect the current datagram's boundary - * in case \c mbedtls_ssl_read has written the maximal - * amount of data fitting into the input buffer. - * - */ -size_t mbedtls_ssl_get_bytes_avail(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the result of the certificate verification - * - * \param ssl The SSL context to use. - * - * \return \c 0 if the certificate verification was successful. - * \return \c -1u if the result is not available. This may happen - * e.g. if the handshake aborts early, or a verification - * callback returned a fatal error. - * \return A bitwise combination of \c MBEDTLS_X509_BADCERT_XXX - * and \c MBEDTLS_X509_BADCRL_XXX failure flags; see x509.h. - */ -uint32_t mbedtls_ssl_get_verify_result(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the id of the current ciphersuite - * - * \param ssl SSL context - * - * \return a ciphersuite id - */ -int mbedtls_ssl_get_ciphersuite_id_from_ssl(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the name of the current ciphersuite - * - * \param ssl SSL context - * - * \return a string containing the ciphersuite name - */ -const char *mbedtls_ssl_get_ciphersuite(const mbedtls_ssl_context *ssl); - - -/** - * \brief Return the (D)TLS protocol version negotiated in the - * given connection. - * - * \note If you call this function too early during the initial - * handshake, before the two sides have agreed on a version, - * this function returns #MBEDTLS_SSL_VERSION_UNKNOWN. - * - * \param ssl The SSL context to query. - * \return The negotiated protocol version. - */ -static inline mbedtls_ssl_protocol_version mbedtls_ssl_get_version_number( - const mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(tls_version); -} - -/** - * \brief Return the current TLS version - * - * \param ssl SSL context - * - * \return a string containing the TLS version - */ -const char *mbedtls_ssl_get_version(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the (maximum) number of bytes added by the record - * layer: header + encryption/MAC overhead (inc. padding) - * - * \param ssl SSL context - * - * \return Current maximum record expansion in bytes - */ -int mbedtls_ssl_get_record_expansion(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the current maximum outgoing record payload in bytes. - * - * \note The logic to determine the maximum outgoing record payload is - * version-specific. It takes into account various factors, such as - * the mbedtls_config.h setting \c MBEDTLS_SSL_OUT_CONTENT_LEN, extensions - * such as the max fragment length or record size limit extension if - * used, and for DTLS the path MTU as configured and current - * record expansion. - * - * \note With DTLS, \c mbedtls_ssl_write() will return an error if - * called with a larger length value. - * With TLS, \c mbedtls_ssl_write() will fragment the input if - * necessary and return the number of bytes written; it is up - * to the caller to call \c mbedtls_ssl_write() again in - * order to send the remaining bytes if any. - * - * \sa mbedtls_ssl_get_max_out_record_payload() - * \sa mbedtls_ssl_get_record_expansion() - * - * \param ssl SSL context - * - * \return Current maximum payload for an outgoing record, - * or a negative error code. - */ -int mbedtls_ssl_get_max_out_record_payload(const mbedtls_ssl_context *ssl); - -/** - * \brief Return the current maximum incoming record payload in bytes. - * - * \note The logic to determine the maximum incoming record payload is - * version-specific. It takes into account various factors, such as - * the mbedtls_config.h setting \c MBEDTLS_SSL_IN_CONTENT_LEN, extensions - * such as the max fragment length extension or record size limit - * extension if used, and the current record expansion. - * - * \sa mbedtls_ssl_set_mtu() - * \sa mbedtls_ssl_get_max_in_record_payload() - * \sa mbedtls_ssl_get_record_expansion() - * - * \param ssl SSL context - * - * \return Current maximum payload for an incoming record, - * or a negative error code. - */ -int mbedtls_ssl_get_max_in_record_payload(const mbedtls_ssl_context *ssl); - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * \brief Return the peer certificate from the current connection. - * - * \param ssl The SSL context to use. This must be initialized and setup. - * - * \return The current peer certificate, if available. - * The returned certificate is owned by the SSL context and - * is valid only until the next call to the SSL API. - * \return \c NULL if no peer certificate is available. This might - * be because the chosen ciphersuite doesn't use CRTs - * (PSK-based ciphersuites, for example), or because - * #MBEDTLS_SSL_KEEP_PEER_CERTIFICATE has been disabled, - * allowing the stack to free the peer's CRT to save memory. - * - * \note For one-time inspection of the peer's certificate during - * the handshake, consider registering an X.509 CRT verification - * callback through mbedtls_ssl_conf_verify() instead of calling - * this function. Using mbedtls_ssl_conf_verify() also comes at - * the benefit of allowing you to influence the verification - * process, for example by masking expected and tolerated - * verification failures. - * - * \warning You must not use the pointer returned by this function - * after any further call to the SSL API, including - * mbedtls_ssl_read() and mbedtls_ssl_write(); this is - * because the pointer might change during renegotiation, - * which happens transparently to the user. - * If you want to use the certificate across API calls, - * you must make a copy. - */ -const mbedtls_x509_crt *mbedtls_ssl_get_peer_cert(const mbedtls_ssl_context *ssl); -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -#if defined(MBEDTLS_SSL_CLI_C) -/** - * \brief Export a session in order to resume it later. - * - * \param ssl The SSL context representing the connection for which to - * to export a session structure for later resumption. - * \param session The target structure in which to store the exported session. - * This must have been initialized with mbedtls_ssl_session_init() - * but otherwise be unused. - * - * \note This function can handle a variety of mechanisms for session - * resumption: For TLS 1.2, both session ID-based resumption and - * ticket-based resumption will be considered. For TLS 1.3, - * sessions equate to tickets, and if session tickets are - * enabled (see #MBEDTLS_SSL_SESSION_TICKETS configuration - * option), this function exports the last received ticket and - * the exported session may be used to resume the TLS 1.3 - * session. If session tickets are disabled, exported sessions - * cannot be used to resume a TLS 1.3 session. - * - * \return \c 0 if successful. In this case, \p session can be used for - * session resumption by passing it to mbedtls_ssl_set_session(), - * and serialized for storage via mbedtls_ssl_session_save(). - * \return Another negative error code on other kinds of failure. - * - * \sa mbedtls_ssl_set_session() - * \sa mbedtls_ssl_session_save() - */ -int mbedtls_ssl_get_session(const mbedtls_ssl_context *ssl, - mbedtls_ssl_session *session); -#endif /* MBEDTLS_SSL_CLI_C */ - -/** - * \brief Perform the SSL handshake - * - * \param ssl SSL context - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_WANT_READ or #MBEDTLS_ERR_SSL_WANT_WRITE - * if the handshake is incomplete and waiting for data to - * be available for reading from or writing to the underlying - * transport - in this case you must call this function again - * when the underlying transport is ready for the operation. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if an asynchronous - * operation is in progress (see - * mbedtls_ssl_conf_async_private_cb()) - in this case you - * must call this function again when the operation is ready. - * \return #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS if a cryptographic - * operation is in progress (see mbedtls_ecp_set_max_ops()) - - * in this case you must call this function again to complete - * the handshake when you're done attending other tasks. - * \return #MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED if DTLS is in use - * and the client did not demonstrate reachability yet - in - * this case you must stop using the context (see below). - * \return #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET if a TLS 1.3 - * NewSessionTicket message has been received. See the - * documentation of mbedtls_ssl_read() for more information - * about this error code. - * \return #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA if early data, as - * defined in RFC 8446 (TLS 1.3 specification), has been - * received as part of the handshake. This is server specific - * and may occur only if the early data feature has been - * enabled on server (see mbedtls_ssl_conf_early_data() - * documentation). You must call mbedtls_ssl_read_early_data() - * to read the early data before resuming the handshake. - * \return Another SSL error code - in this case you must stop using - * the context (see below). - * - * \warning If this function returns something other than - * \c 0, - * #MBEDTLS_ERR_SSL_WANT_READ, - * #MBEDTLS_ERR_SSL_WANT_WRITE, - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS or - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS or - * #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET or - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA, - * you must stop using the SSL context for reading or writing, - * and either free it or call \c mbedtls_ssl_session_reset() - * on it before re-using it for a new connection; the current - * connection must be closed. - * - * \note If DTLS is in use, then you may choose to handle - * #MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED specially for logging - * purposes, as it is an expected return value rather than an - * actual error, but you still need to reset/free the context. - * - * \note Remarks regarding event-driven DTLS: - * If the function returns #MBEDTLS_ERR_SSL_WANT_READ, no datagram - * from the underlying transport layer is currently being processed, - * and it is safe to idle until the timer or the underlying transport - * signal a new event. This is not true for a successful handshake, - * in which case the datagram of the underlying transport that is - * currently being processed might or might not contain further - * DTLS records. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * Otherwise, the handshake may call psa_crypto_init() - * if a negotiation involving TLS 1.3 takes place (this may - * be the case even if TLS 1.3 is offered but eventually - * not selected). - * - * \note In TLS, reception of fragmented handshake messages is - * supported with some limitations (those limitations do - * not apply to DTLS, where defragmentation is fully - * supported): - * - On an Mbed TLS server that only accepts TLS 1.2, - * the initial ClientHello message must not be fragmented. - * A TLS 1.2 ClientHello may be fragmented if the server - * also accepts TLS 1.3 connections (meaning - * that #MBEDTLS_SSL_PROTO_TLS1_3 enabled, and the - * accepted versions have not been restricted with - * mbedtls_ssl_conf_max_tls_version() or the like). - * - The first fragment of a handshake message must be - * at least 4 bytes long. - * - Non-handshake records must not be interleaved between - * the fragments of a handshake message. (This is permitted - * in TLS 1.2 but not in TLS 1.3, but Mbed TLS rejects it - * even in TLS 1.2.) - */ -int mbedtls_ssl_handshake(mbedtls_ssl_context *ssl); - -/** - * \brief After calling mbedtls_ssl_handshake() to start the SSL - * handshake you can call this function to check whether the - * handshake is over for a given SSL context. This function - * should be also used to determine when to stop calling - * mbedtls_handshake_step() for that context. - * - * \param ssl SSL context - * - * \return \c 1 if handshake is over, \c 0 if it is still ongoing. - */ -static inline int mbedtls_ssl_is_handshake_over(mbedtls_ssl_context *ssl) -{ - return ssl->MBEDTLS_PRIVATE(state) >= MBEDTLS_SSL_HANDSHAKE_OVER; -} - -/** - * \brief Perform a single step of the SSL handshake - * - * \note The state of the context (ssl->state) will be at - * the next state after this function returns \c 0. Do not - * call this function if mbedtls_ssl_is_handshake_over() - * returns \c 1. - * - * \warning Whilst in the past you may have used direct access to the - * context state (ssl->state) in order to ascertain when to - * stop calling this function and although you can still do - * so with something like ssl->MBEDTLS_PRIVATE(state) or by - * defining MBEDTLS_ALLOW_PRIVATE_ACCESS, this is now - * considered deprecated and could be broken in any future - * release. If you still find you have good reason for such - * direct access, then please do contact the team to explain - * this (raise an issue or post to the mailing list), so that - * we can add a solution to your problem that will be - * guaranteed to work in the future. - * - * \param ssl SSL context - * - * \return See mbedtls_ssl_handshake(). - * - * \warning If this function returns something other than \c 0, - * #MBEDTLS_ERR_SSL_WANT_READ, #MBEDTLS_ERR_SSL_WANT_WRITE, - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS, - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS or - * #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET or - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA, you must stop using - * the SSL context for reading or writing, and either free it - * or call \c mbedtls_ssl_session_reset() on it before - * re-using it for a new connection; the current connection - * must be closed. - */ -int mbedtls_ssl_handshake_step(mbedtls_ssl_context *ssl); - -#if defined(MBEDTLS_SSL_RENEGOTIATION) -/** - * \brief Initiate an SSL renegotiation on the running connection. - * Client: perform the renegotiation right now. - * Server: request renegotiation, which will be performed - * during the next call to mbedtls_ssl_read() if honored by - * client. - * - * \param ssl SSL context - * - * \return 0 if successful, or any mbedtls_ssl_handshake() return - * value except #MBEDTLS_ERR_SSL_CLIENT_RECONNECT that can't - * happen during a renegotiation. - * - * \warning If this function returns something other than \c 0, - * #MBEDTLS_ERR_SSL_WANT_READ, #MBEDTLS_ERR_SSL_WANT_WRITE, - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS or - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS, you must stop using - * the SSL context for reading or writing, and either free it - * or call \c mbedtls_ssl_session_reset() on it before - * re-using it for a new connection; the current connection - * must be closed. - * - */ -int mbedtls_ssl_renegotiate(mbedtls_ssl_context *ssl); -#endif /* MBEDTLS_SSL_RENEGOTIATION */ - -/** - * \brief Read at most 'len' application data bytes - * - * \param ssl SSL context - * \param buf buffer that will hold the data - * \param len maximum number of bytes to read - * - * \return The (positive) number of bytes read if successful. - * \return \c 0 if the read end of the underlying transport was closed - * without sending a CloseNotify beforehand, which might happen - * because of various reasons (internal error of an underlying - * stack, non-conformant peer not sending a CloseNotify and - * such) - in this case you must stop using the context - * (see below). - * \return #MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY if the underlying - * transport is still functional, but the peer has - * acknowledged to not send anything anymore. - * \return #MBEDTLS_ERR_SSL_WANT_READ or #MBEDTLS_ERR_SSL_WANT_WRITE - * if the handshake is incomplete and waiting for data to - * be available for reading from or writing to the underlying - * transport - in this case you must call this function again - * when the underlying transport is ready for the operation. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if an asynchronous - * operation is in progress (see - * mbedtls_ssl_conf_async_private_cb()) - in this case you - * must call this function again when the operation is ready. - * \return #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS if a cryptographic - * operation is in progress (see mbedtls_ecp_set_max_ops()) - - * in this case you must call this function again to complete - * the handshake when you're done attending other tasks. - * \return #MBEDTLS_ERR_SSL_CLIENT_RECONNECT if we're at the server - * side of a DTLS connection and the client is initiating a - * new connection using the same source port. See below. - * \return #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET if a TLS 1.3 - * NewSessionTicket message has been received. - * This error code is only returned on the client side. It is - * only returned if handling of TLS 1.3 NewSessionTicket - * messages has been enabled through - * mbedtls_ssl_conf_tls13_enable_signal_new_session_tickets(). - * This error code indicates that a TLS 1.3 NewSessionTicket - * message has been received and parsed successfully by the - * client. The ticket data can be retrieved from the SSL - * context by calling mbedtls_ssl_get_session(). It remains - * available until the next call to mbedtls_ssl_read(). - * \return #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA if early data, as - * defined in RFC 8446 (TLS 1.3 specification), has been - * received as part of the handshake. This is server specific - * and may occur only if the early data feature has been - * enabled on server (see mbedtls_ssl_conf_early_data() - * documentation). You must call mbedtls_ssl_read_early_data() - * to read the early data before resuming the handshake. - * \return Another SSL error code - in this case you must stop using - * the context (see below). - * - * \warning If this function returns something other than - * a positive value, - * #MBEDTLS_ERR_SSL_WANT_READ, - * #MBEDTLS_ERR_SSL_WANT_WRITE, - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS, - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS, - * #MBEDTLS_ERR_SSL_CLIENT_RECONNECT or - * #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET or - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA, - * you must stop using the SSL context for reading or writing, - * and either free it or call \c mbedtls_ssl_session_reset() - * on it before re-using it for a new connection; the current - * connection must be closed. - * - * \note When this function returns #MBEDTLS_ERR_SSL_CLIENT_RECONNECT - * (which can only happen server-side), it means that a client - * is initiating a new connection using the same source port. - * You can either treat that as a connection close and wait - * for the client to resend a ClientHello, or directly - * continue with \c mbedtls_ssl_handshake() with the same - * context (as it has been reset internally). Either way, you - * must make sure this is seen by the application as a new - * connection: application state, if any, should be reset, and - * most importantly the identity of the client must be checked - * again. WARNING: not validating the identity of the client - * again, or not transmitting the new identity to the - * application layer, would allow authentication bypass! - * - * \note Remarks regarding event-driven DTLS: - * - If the function returns #MBEDTLS_ERR_SSL_WANT_READ, no datagram - * from the underlying transport layer is currently being processed, - * and it is safe to idle until the timer or the underlying transport - * signal a new event. - * - This function may return MBEDTLS_ERR_SSL_WANT_READ even if data was - * initially available on the underlying transport, as this data may have - * been only e.g. duplicated messages or a renegotiation request. - * Therefore, you must be prepared to receive MBEDTLS_ERR_SSL_WANT_READ even - * when reacting to an incoming-data event from the underlying transport. - * - On success, the datagram of the underlying transport that is currently - * being processed may contain further DTLS records. You should call - * \c mbedtls_ssl_check_pending to check for remaining records. - * - */ -int mbedtls_ssl_read(mbedtls_ssl_context *ssl, unsigned char *buf, size_t len); - -/** - * \brief Try to write exactly 'len' application data bytes - * - * \warning This function will do partial writes in some cases. If the - * return value is non-negative but less than length, the - * function must be called again with updated arguments: - * buf + ret, len - ret (if ret is the return value) until - * it returns a value equal to the last 'len' argument. - * - * \param ssl SSL context - * \param buf buffer holding the data - * \param len how many bytes must be written - * - * \return The (non-negative) number of bytes actually written if - * successful (may be less than \p len). - * \return #MBEDTLS_ERR_SSL_WANT_READ or #MBEDTLS_ERR_SSL_WANT_WRITE - * if the handshake is incomplete and waiting for data to - * be available for reading from or writing to the underlying - * transport - in this case you must call this function again - * when the underlying transport is ready for the operation. - * \return #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS if an asynchronous - * operation is in progress (see - * mbedtls_ssl_conf_async_private_cb()) - in this case you - * must call this function again when the operation is ready. - * \return #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS if a cryptographic - * operation is in progress (see mbedtls_ecp_set_max_ops()) - - * in this case you must call this function again to complete - * the handshake when you're done attending other tasks. - * \return #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET if a TLS 1.3 - * NewSessionTicket message has been received. See the - * documentation of mbedtls_ssl_read() for more information - * about this error code. - * \return #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA if early data, as - * defined in RFC 8446 (TLS 1.3 specification), has been - * received as part of the handshake. This is server specific - * and may occur only if the early data feature has been - * enabled on server (see mbedtls_ssl_conf_early_data() - * documentation). You must call mbedtls_ssl_read_early_data() - * to read the early data before resuming the handshake. - * \return Another SSL error code - in this case you must stop using - * the context (see below). - * - * \warning If this function returns something other than - * a non-negative value, - * #MBEDTLS_ERR_SSL_WANT_READ, - * #MBEDTLS_ERR_SSL_WANT_WRITE, - * #MBEDTLS_ERR_SSL_ASYNC_IN_PROGRESS, - * #MBEDTLS_ERR_SSL_CRYPTO_IN_PROGRESS or - * #MBEDTLS_ERR_SSL_RECEIVED_NEW_SESSION_TICKET or - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA, - * you must stop using the SSL context for reading or writing, - * and either free it or call \c mbedtls_ssl_session_reset() - * on it before re-using it for a new connection; the current - * connection must be closed. - * - * \note When this function returns #MBEDTLS_ERR_SSL_WANT_WRITE/READ, - * it must be called later with the *same* arguments, - * until it returns a value greater than or equal to 0. When - * the function returns #MBEDTLS_ERR_SSL_WANT_WRITE there may be - * some partial data in the output buffer, however this is not - * yet sent. - * - * \note If the requested length is greater than the maximum - * fragment length (either the built-in limit or the one set - * or negotiated with the peer), then: - * - with TLS, less bytes than requested are written. - * - with DTLS, MBEDTLS_ERR_SSL_BAD_INPUT_DATA is returned. - * \c mbedtls_ssl_get_max_out_record_payload() may be used to - * query the active maximum fragment length. - * - * \note Attempting to write 0 bytes will result in an empty TLS - * application record being sent. - */ -int mbedtls_ssl_write(mbedtls_ssl_context *ssl, const unsigned char *buf, size_t len); - -/** - * \brief Send an alert message - * - * \param ssl SSL context - * \param level The alert level of the message - * (MBEDTLS_SSL_ALERT_LEVEL_WARNING or MBEDTLS_SSL_ALERT_LEVEL_FATAL) - * \param message The alert message (SSL_ALERT_MSG_*) - * - * \return 0 if successful, or a specific SSL error code. - * - * \note If this function returns something other than 0 or - * MBEDTLS_ERR_SSL_WANT_READ/WRITE, you must stop using - * the SSL context for reading or writing, and either free it or - * call \c mbedtls_ssl_session_reset() on it before re-using it - * for a new connection; the current connection must be closed. - */ -int mbedtls_ssl_send_alert_message(mbedtls_ssl_context *ssl, - unsigned char level, - unsigned char message); -/** - * \brief Notify the peer that the connection is being closed - * - * \param ssl SSL context - * - * \return 0 if successful, or a specific SSL error code. - * - * \note If this function returns something other than 0 or - * MBEDTLS_ERR_SSL_WANT_READ/WRITE, you must stop using - * the SSL context for reading or writing, and either free it or - * call \c mbedtls_ssl_session_reset() on it before re-using it - * for a new connection; the current connection must be closed. - */ -int mbedtls_ssl_close_notify(mbedtls_ssl_context *ssl); - -#if defined(MBEDTLS_SSL_EARLY_DATA) - -#if defined(MBEDTLS_SSL_SRV_C) -/** - * \brief Read at most 'len' bytes of early data - * - * \note This API is server specific. - * - * \warning Early data is defined in the TLS 1.3 specification, RFC 8446. - * IMPORTANT NOTE from section 2.3 of the specification: - * - * The security properties for 0-RTT data are weaker than - * those for other kinds of TLS data. Specifically: - * - This data is not forward secret, as it is encrypted - * solely under keys derived using the offered PSK. - * - There are no guarantees of non-replay between connections. - * Protection against replay for ordinary TLS 1.3 1-RTT data - * is provided via the server's Random value, but 0-RTT data - * does not depend on the ServerHello and therefore has - * weaker guarantees. This is especially relevant if the - * data is authenticated either with TLS client - * authentication or inside the application protocol. The - * same warnings apply to any use of the - * early_exporter_master_secret. - * - * \warning Mbed TLS does not implement any of the anti-replay defenses - * defined in section 8 of the TLS 1.3 specification: - * single-use of tickets or ClientHello recording within a - * given time window. - * - * \note This function is used in conjunction with - * mbedtls_ssl_handshake(), mbedtls_ssl_handshake_step(), - * mbedtls_ssl_read() and mbedtls_ssl_write() to read early - * data when these functions return - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA. - * - * \param ssl SSL context, it must have been initialized and set up. - * \param buf buffer that will hold the data - * \param len maximum number of bytes to read - * - * \return The (positive) number of bytes read if successful. - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if input data is invalid. - * \return #MBEDTLS_ERR_SSL_CANNOT_READ_EARLY_DATA if it is not - * possible to read early data for the SSL context \p ssl. Note - * that this function is intended to be called for an SSL - * context \p ssl only after a call to mbedtls_ssl_handshake(), - * mbedtls_ssl_handshake_step(), mbedtls_ssl_read() or - * mbedtls_ssl_write() for \p ssl that has returned - * #MBEDTLS_ERR_SSL_RECEIVED_EARLY_DATA. - */ -int mbedtls_ssl_read_early_data(mbedtls_ssl_context *ssl, - unsigned char *buf, size_t len); -#endif /* MBEDTLS_SSL_SRV_C */ - -#if defined(MBEDTLS_SSL_CLI_C) -/** - * \brief Try to write exactly 'len' application data bytes while - * performing the handshake (early data). - * - * \warning Early data is defined in the TLS 1.3 specification, RFC 8446. - * IMPORTANT NOTE from section 2.3 of the specification: - * - * The security properties for 0-RTT data are weaker than - * those for other kinds of TLS data. Specifically: - * - This data is not forward secret, as it is encrypted - * solely under keys derived using the offered PSK. - * - There are no guarantees of non-replay between connections. - * Protection against replay for ordinary TLS 1.3 1-RTT data - * is provided via the server's Random value, but 0-RTT data - * does not depend on the ServerHello and therefore has - * weaker guarantees. This is especially relevant if the - * data is authenticated either with TLS client - * authentication or inside the application protocol. The - * same warnings apply to any use of the - * early_exporter_master_secret. - * - * \note This function behaves mainly as mbedtls_ssl_write(). The - * specification of mbedtls_ssl_write() relevant to TLS 1.3 - * (thus not the parts specific to (D)TLS1.2) applies to this - * function and the present documentation is mainly restricted - * to the differences with mbedtls_ssl_write(). One noticeable - * difference though is that mbedtls_ssl_write() aims to - * complete the handshake before to write application data - * while mbedtls_ssl_write_early() aims to drive the handshake - * just past the point where it is not possible to send early - * data anymore. - * - * \param ssl SSL context - * \param buf buffer holding the data - * \param len how many bytes must be written - * - * \return The (non-negative) number of bytes actually written if - * successful (may be less than \p len). - * - * \return One additional specific error code compared to - * mbedtls_ssl_write(): - * #MBEDTLS_ERR_SSL_CANNOT_WRITE_EARLY_DATA. - * - * #MBEDTLS_ERR_SSL_CANNOT_WRITE_EARLY_DATA is returned when it - * is not possible to write early data for the SSL context - * \p ssl. - * - * It may have been possible and it is not possible - * anymore because the client received the server Finished - * message, the server rejected early data or the maximum - * number of allowed early data for the PSK in use has been - * reached. - * - * It may never have been possible and will never be possible - * for the SSL context \p ssl because the use of early data - * is disabled for that context or more generally the context - * is not suitably configured to enable early data or the first - * call to the function was done while the handshake was - * already completed. - * - * It is not possible to write early data for the SSL context - * \p ssl and any subsequent call to this API will return this - * error code. But this does not preclude for using it with - * mbedtls_ssl_write(), mbedtls_ssl_read() or - * mbedtls_ssl_handshake() and the handshake can be - * completed by calling one of these APIs. - * - * \note This function may write early data only if the SSL context - * has been configured for the handshake with a PSK for which - * early data is allowed. - * - * \note To maximize the number of early data that can be written in - * the course of the handshake, it is expected that this - * function starts the handshake for the SSL context \p ssl. - * But this is not mandatory. - * - * \note This function does not provide any information on whether - * the server has accepted or will accept early data or not. - * When it returns a positive value, it just means that it - * has written early data to the server. To know whether the - * server has accepted early data or not, you should call - * mbedtls_ssl_get_early_data_status() with the handshake - * completed. - */ -int mbedtls_ssl_write_early_data(mbedtls_ssl_context *ssl, - const unsigned char *buf, size_t len); - -/** - * \brief Get the status of the negotiation of the use of early data. - * - * \param ssl The SSL context to query - * - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if this function is called - * from the server-side. - * - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if this function is called - * prior to completion of the handshake. - * - * \return #MBEDTLS_SSL_EARLY_DATA_STATUS_NOT_INDICATED if the client - * has not indicated the use of early data to the server. - * - * \return #MBEDTLS_SSL_EARLY_DATA_STATUS_ACCEPTED if the client has - * indicated the use of early data and the server has accepted - * it. - * - * \return #MBEDTLS_SSL_EARLY_DATA_STATUS_REJECTED if the client has - * indicated the use of early data but the server has rejected - * it. In this situation, the client may want to re-send the - * early data it may have tried to send by calling - * mbedtls_ssl_write_early_data() as ordinary post-handshake - * application data by calling mbedtls_ssl_write(). - * - */ -int mbedtls_ssl_get_early_data_status(mbedtls_ssl_context *ssl); -#endif /* MBEDTLS_SSL_CLI_C */ - -#endif /* MBEDTLS_SSL_EARLY_DATA */ - -/** - * \brief Free referenced items in an SSL context and clear memory - * - * \param ssl SSL context - */ -void mbedtls_ssl_free(mbedtls_ssl_context *ssl); - -#if defined(MBEDTLS_SSL_CONTEXT_SERIALIZATION) -/** - * \brief Save an active connection as serialized data in a buffer. - * This allows the freeing or re-using of the SSL context - * while still picking up the connection later in a way that - * it entirely transparent to the peer. - * - * \see mbedtls_ssl_context_load() - * - * \note The serialized data only contains the data that is - * necessary to resume the connection: negotiated protocol - * options, session identifier, keys, etc. - * Loading a saved SSL context does not restore settings and - * state related to how the application accesses the context, - * such as configured callback functions, user data, pending - * incoming or outgoing data, etc. - * - * \note This feature is currently only available under certain - * conditions, see the documentation of the return value - * #MBEDTLS_ERR_SSL_BAD_INPUT_DATA for details. - * - * \note When this function succeeds, it calls - * mbedtls_ssl_session_reset() on \p ssl which as a result is - * no longer associated with the connection that has been - * serialized. This avoids creating copies of the connection - * state. You're then free to either re-use the context - * structure for a different connection, or call - * mbedtls_ssl_free() on it. See the documentation of - * mbedtls_ssl_session_reset() for more details. - * - * \param ssl The SSL context to save. On success, it is no longer - * associated with the connection that has been serialized. - * \param buf The buffer to write the serialized data to. It must be a - * writeable buffer of at least \p buf_len bytes, or may be \c - * NULL if \p buf_len is \c 0. - * \param buf_len The number of bytes available for writing in \p buf. - * \param olen The size in bytes of the data that has been or would have - * been written. It must point to a valid \c size_t. - * - * \note \p olen is updated to the correct value regardless of - * whether \p buf_len was large enough. This makes it possible - * to determine the necessary size by calling this function - * with \p buf set to \c NULL and \p buf_len to \c 0. However, - * the value of \p olen is only guaranteed to be correct when - * the function returns #MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL or - * \c 0. If the return value is different, then the value of - * \p olen is undefined. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL if \p buf is too small. - * \return #MBEDTLS_ERR_SSL_ALLOC_FAILED if memory allocation failed - * while resetting the context. - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if a handshake is in - * progress, or there is pending data for reading or sending, - * or the connection does not use DTLS 1.2 with an AEAD - * ciphersuite, or renegotiation is enabled. - */ -int mbedtls_ssl_context_save(mbedtls_ssl_context *ssl, - unsigned char *buf, - size_t buf_len, - size_t *olen); - -/** - * \brief Load serialized connection data to an SSL context. - * - * \see mbedtls_ssl_context_save() - * - * \warning The same serialized data must never be loaded into more - * that one context. In order to ensure that, after - * successfully loading serialized data to an SSL context, you - * should immediately destroy or invalidate all copies of the - * serialized data that was loaded. Loading the same data in - * more than one context would cause severe security failures - * including but not limited to loss of confidentiality. - * - * \note Before calling this function, the SSL context must be - * prepared in one of the two following ways. The first way is - * to take a context freshly initialised with - * mbedtls_ssl_init() and call mbedtls_ssl_setup() on it with - * the same ::mbedtls_ssl_config structure that was used in - * the original connection. The second way is to - * call mbedtls_ssl_session_reset() on a context that was - * previously prepared as above but used in the meantime. - * Either way, you must not use the context to perform a - * handshake between calling mbedtls_ssl_setup() or - * mbedtls_ssl_session_reset() and calling this function. You - * may however call other setter functions in that time frame - * as indicated in the note below. - * - * \note Before or after calling this function successfully, you - * also need to configure some connection-specific callbacks - * and settings before you can use the connection again - * (unless they were already set before calling - * mbedtls_ssl_session_reset() and the values are suitable for - * the present connection). Specifically, you want to call - * at least mbedtls_ssl_set_bio(), - * mbedtls_ssl_set_timer_cb(), and - * mbedtls_ssl_set_user_data_n() or - * mbedtls_ssl_set_user_data_p() if they were set originally. - * All other SSL setter functions - * are not necessary to call, either because they're only used - * in handshakes, or because the setting is already saved. You - * might choose to call them anyway, for example in order to - * share code between the cases of establishing a new - * connection and the case of loading an already-established - * connection. - * - * \note If you have new information about the path MTU, you want to - * call mbedtls_ssl_set_mtu() after calling this function, as - * otherwise this function would overwrite your - * newly-configured value with the value that was active when - * the context was saved. - * - * \note When this function returns an error code, it calls - * mbedtls_ssl_free() on \p ssl. In this case, you need to - * prepare the context with the usual sequence starting with a - * call to mbedtls_ssl_init() if you want to use it again. - * - * \param ssl The SSL context structure to be populated. It must have - * been prepared as described in the note above. - * \param buf The buffer holding the serialized connection data. It must - * be a readable buffer of at least \p len bytes. - * \param len The size of the serialized data in bytes. - * - * \return \c 0 if successful. - * \return #MBEDTLS_ERR_SSL_ALLOC_FAILED if memory allocation failed. - * \return #MBEDTLS_ERR_SSL_VERSION_MISMATCH if the serialized data - * comes from a different Mbed TLS version or build. - * \return #MBEDTLS_ERR_SSL_BAD_INPUT_DATA if input data is invalid. - */ -int mbedtls_ssl_context_load(mbedtls_ssl_context *ssl, - const unsigned char *buf, - size_t len); -#endif /* MBEDTLS_SSL_CONTEXT_SERIALIZATION */ - -/** - * \brief Initialize an SSL configuration context - * Just makes the context ready for - * mbedtls_ssl_config_defaults() or mbedtls_ssl_config_free(). - * - * \note You need to call mbedtls_ssl_config_defaults() unless you - * manually set all of the relevant fields yourself. - * - * \param conf SSL configuration context - */ -void mbedtls_ssl_config_init(mbedtls_ssl_config *conf); - -/** - * \brief Load reasonable default SSL configuration values. - * (You need to call mbedtls_ssl_config_init() first.) - * - * \param conf SSL configuration context - * \param endpoint MBEDTLS_SSL_IS_CLIENT or MBEDTLS_SSL_IS_SERVER - * \param transport MBEDTLS_SSL_TRANSPORT_STREAM for TLS, or - * MBEDTLS_SSL_TRANSPORT_DATAGRAM for DTLS - * \param preset a MBEDTLS_SSL_PRESET_XXX value - * - * \note See \c mbedtls_ssl_conf_transport() for notes on DTLS. - * - * \return 0 if successful, or - * MBEDTLS_ERR_XXX_ALLOC_FAILED on memory allocation error. - */ -int mbedtls_ssl_config_defaults(mbedtls_ssl_config *conf, - int endpoint, int transport, int preset); - -/** - * \brief Free an SSL configuration context - * - * \param conf SSL configuration context - */ -void mbedtls_ssl_config_free(mbedtls_ssl_config *conf); - -/** - * \brief Initialize SSL session structure - * - * \param session SSL session - */ -void mbedtls_ssl_session_init(mbedtls_ssl_session *session); - -/** - * \brief Free referenced items in an SSL session including the - * peer certificate and clear memory - * - * \note A session object can be freed even if the SSL context - * that was used to retrieve the session is still in use. - * - * \param session SSL session - */ -void mbedtls_ssl_session_free(mbedtls_ssl_session *session); - -/** - * \brief TLS-PRF function for key derivation. - * - * \param prf The tls_prf type function type to be used. - * \param secret Secret for the key derivation function. - * \param slen Length of the secret. - * \param label String label for the key derivation function, - * terminated with null character. - * \param random Random bytes. - * \param rlen Length of the random bytes buffer. - * \param dstbuf The buffer holding the derived key. - * \param dlen Length of the output buffer. - * - * \return 0 on success. An SSL specific error on failure. - */ -int mbedtls_ssl_tls_prf(const mbedtls_tls_prf_types prf, - const unsigned char *secret, size_t slen, - const char *label, - const unsigned char *random, size_t rlen, - unsigned char *dstbuf, size_t dlen); - -#if defined(MBEDTLS_SSL_KEYING_MATERIAL_EXPORT) -/* Maximum value for key_len in mbedtls_ssl_export_keying material. Depending on the TLS - * version and the negotiated ciphersuite, larger keys could in principle be exported, - * but for simplicity, we define one limit that works in all cases. TLS 1.3 with SHA256 - * has the strictest limit: 255 blocks of SHA256 output, or 8160 bytes. */ -#define MBEDTLS_SSL_EXPORT_MAX_KEY_LEN 8160 - -/** - * \brief TLS-Exporter to derive shared symmetric keys between server and client. - * - * \param ssl SSL context from which to export keys. Must have finished the handshake. - * \param out Output buffer of length at least key_len bytes. - * \param key_len Length of the key to generate in bytes, must be at most - * MBEDTLS_SSL_EXPORT_MAX_KEY_LEN (8160). - * \param label Label for which to generate the key of length label_len. - * \param label_len Length of label in bytes. Must be at most 249 in TLS 1.3. - * \param context Context of the key. Can be NULL if context_len or use_context is 0. - * \param context_len Length of context. Must be < 2^16 in TLS 1.2. - * \param use_context Indicates if a context should be used in deriving the key. - * - * \note TLS 1.2 makes a distinction between a 0-length context and no context. - * This is why the use_context argument exists. TLS 1.3 does not make - * this distinction. If use_context is 0 and TLS 1.3 is used, context and - * context_len are ignored and a 0-length context is used. - * - * \return 0 on success. - * \return MBEDTLS_ERR_SSL_BAD_INPUT_DATA if the handshake is not yet completed. - * \return An SSL-specific error on failure. - */ -int mbedtls_ssl_export_keying_material(mbedtls_ssl_context *ssl, - uint8_t *out, const size_t key_len, - const char *label, const size_t label_len, - const unsigned char *context, const size_t context_len, - const int use_context); -#endif -#ifdef __cplusplus -} -#endif - -#endif /* ssl.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cache.h b/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cache.h deleted file mode 100644 index a1307b450..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cache.h +++ /dev/null @@ -1,187 +0,0 @@ -/** - * \file ssl_cache.h - * - * \brief SSL session cache implementation - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SSL_CACHE_H -#define MBEDTLS_SSL_CACHE_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/ssl.h" - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ - -#if !defined(MBEDTLS_SSL_CACHE_DEFAULT_TIMEOUT) -#define MBEDTLS_SSL_CACHE_DEFAULT_TIMEOUT 86400 /*!< 1 day */ -#endif - -#if !defined(MBEDTLS_SSL_CACHE_DEFAULT_MAX_ENTRIES) -#define MBEDTLS_SSL_CACHE_DEFAULT_MAX_ENTRIES 50 /*!< Maximum entries in cache */ -#endif - -/** \} name SECTION: Module settings */ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct mbedtls_ssl_cache_context mbedtls_ssl_cache_context; -typedef struct mbedtls_ssl_cache_entry mbedtls_ssl_cache_entry; - -/** - * \brief This structure is used for storing cache entries - */ -struct mbedtls_ssl_cache_entry { -#if defined(MBEDTLS_HAVE_TIME) - mbedtls_time_t MBEDTLS_PRIVATE(timestamp); /*!< entry timestamp */ -#endif - - unsigned char MBEDTLS_PRIVATE(session_id)[32]; /*!< session ID */ - size_t MBEDTLS_PRIVATE(session_id_len); - - unsigned char *MBEDTLS_PRIVATE(session); /*!< serialized session */ - size_t MBEDTLS_PRIVATE(session_len); - - mbedtls_ssl_cache_entry *MBEDTLS_PRIVATE(next); /*!< chain pointer */ -}; - -/** - * \brief Cache context - */ -struct mbedtls_ssl_cache_context { - mbedtls_ssl_cache_entry *MBEDTLS_PRIVATE(chain); /*!< start of the chain */ - int MBEDTLS_PRIVATE(timeout); /*!< cache entry timeout */ - int MBEDTLS_PRIVATE(max_entries); /*!< maximum entries */ -#if defined(MBEDTLS_THREADING_C) - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); /*!< mutex */ -#endif -}; - -/** - * \brief Initialize an SSL cache context - * - * \param cache SSL cache context - */ -void mbedtls_ssl_cache_init(mbedtls_ssl_cache_context *cache); - -/** - * \brief Cache get callback implementation - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param data The SSL cache context to use. - * \param session_id The pointer to the buffer holding the session ID - * for the session to load. - * \param session_id_len The length of \p session_id in bytes. - * \param session The address at which to store the session - * associated with \p session_id, if present. - * - * \return \c 0 on success. - * \return #MBEDTLS_ERR_SSL_CACHE_ENTRY_NOT_FOUND if there is - * no cache entry with specified session ID found, or - * any other negative error code for other failures. - */ -int mbedtls_ssl_cache_get(void *data, - unsigned char const *session_id, - size_t session_id_len, - mbedtls_ssl_session *session); - -/** - * \brief Cache set callback implementation - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param data The SSL cache context to use. - * \param session_id The pointer to the buffer holding the session ID - * associated to \p session. - * \param session_id_len The length of \p session_id in bytes. - * \param session The session to store. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -int mbedtls_ssl_cache_set(void *data, - unsigned char const *session_id, - size_t session_id_len, - const mbedtls_ssl_session *session); - -/** - * \brief Remove the cache entry by the session ID - * (Thread-safe if MBEDTLS_THREADING_C is enabled) - * - * \param data The SSL cache context to use. - * \param session_id The pointer to the buffer holding the session ID - * associated to session. - * \param session_id_len The length of \p session_id in bytes. - * - * \return \c 0 on success. This indicates the cache entry for - * the session with provided ID is removed or does not - * exist. - * \return A negative error code on failure. - */ -int mbedtls_ssl_cache_remove(void *data, - unsigned char const *session_id, - size_t session_id_len); - -#if defined(MBEDTLS_HAVE_TIME) -/** - * \brief Set the cache timeout - * (Default: MBEDTLS_SSL_CACHE_DEFAULT_TIMEOUT (1 day)) - * - * A timeout of 0 indicates no timeout. - * - * \param cache SSL cache context - * \param timeout cache entry timeout in seconds - */ -void mbedtls_ssl_cache_set_timeout(mbedtls_ssl_cache_context *cache, int timeout); - -/** - * \brief Get the cache timeout - * - * A timeout of 0 indicates no timeout. - * - * \param cache SSL cache context - * - * \return cache entry timeout in seconds - */ -static inline int mbedtls_ssl_cache_get_timeout(mbedtls_ssl_cache_context *cache) -{ - return cache->MBEDTLS_PRIVATE(timeout); -} -#endif /* MBEDTLS_HAVE_TIME */ - -/** - * \brief Set the maximum number of cache entries - * (Default: MBEDTLS_SSL_CACHE_DEFAULT_MAX_ENTRIES (50)) - * - * \param cache SSL cache context - * \param max cache entry maximum - */ -void mbedtls_ssl_cache_set_max_entries(mbedtls_ssl_cache_context *cache, int max); - -/** - * \brief Free referenced items in a cache context and clear memory - * - * \param cache SSL cache context - */ -void mbedtls_ssl_cache_free(mbedtls_ssl_cache_context *cache); - -#ifdef __cplusplus -} -#endif - -#endif /* ssl_cache.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ciphersuites.h b/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ciphersuites.h deleted file mode 100644 index 12d446200..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ciphersuites.h +++ /dev/null @@ -1,482 +0,0 @@ -/** - * \file ssl_ciphersuites.h - * - * \brief SSL Ciphersuites for Mbed TLS - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SSL_CIPHERSUITES_H -#define MBEDTLS_SSL_CIPHERSUITES_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/pk.h" -#include "mbedtls/cipher.h" -#include "mbedtls/md.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Supported ciphersuites (Official IANA names) - */ -#define MBEDTLS_TLS_RSA_WITH_NULL_MD5 0x01 /**< Weak! */ -#define MBEDTLS_TLS_RSA_WITH_NULL_SHA 0x02 /**< Weak! */ - -#define MBEDTLS_TLS_PSK_WITH_NULL_SHA 0x2C /**< Weak! */ -#define MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA 0x2D /**< Weak! */ -#define MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA 0x2E /**< Weak! */ -#define MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA 0x2F - -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA 0x33 -#define MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA 0x35 -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA 0x39 - -#define MBEDTLS_TLS_RSA_WITH_NULL_SHA256 0x3B /**< Weak! */ -#define MBEDTLS_TLS_RSA_WITH_AES_128_CBC_SHA256 0x3C /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_AES_256_CBC_SHA256 0x3D /**< TLS 1.2 */ - -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA 0x41 -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA 0x45 - -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 0x67 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 0x6B /**< TLS 1.2 */ - -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA 0x84 -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA 0x88 - -#define MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA 0x8C -#define MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA 0x8D - -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA 0x90 -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA 0x91 - -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA 0x94 -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA 0x95 - -#define MBEDTLS_TLS_RSA_WITH_AES_128_GCM_SHA256 0x9C /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_AES_256_GCM_SHA384 0x9D /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 0x9E /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 0x9F /**< TLS 1.2 */ - -#define MBEDTLS_TLS_PSK_WITH_AES_128_GCM_SHA256 0xA8 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_AES_256_GCM_SHA384 0xA9 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 0xAA /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 0xAB /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 0xAC /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 0xAD /**< TLS 1.2 */ - -#define MBEDTLS_TLS_PSK_WITH_AES_128_CBC_SHA256 0xAE -#define MBEDTLS_TLS_PSK_WITH_AES_256_CBC_SHA384 0xAF -#define MBEDTLS_TLS_PSK_WITH_NULL_SHA256 0xB0 /**< Weak! */ -#define MBEDTLS_TLS_PSK_WITH_NULL_SHA384 0xB1 /**< Weak! */ - -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 0xB2 -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 0xB3 -#define MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA256 0xB4 /**< Weak! */ -#define MBEDTLS_TLS_DHE_PSK_WITH_NULL_SHA384 0xB5 /**< Weak! */ - -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 0xB6 -#define MBEDTLS_TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 0xB7 -#define MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA256 0xB8 /**< Weak! */ -#define MBEDTLS_TLS_RSA_PSK_WITH_NULL_SHA384 0xB9 /**< Weak! */ - -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 0xBA /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 0xBE /**< TLS 1.2 */ - -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 0xC0 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 0xC4 /**< TLS 1.2 */ - -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_NULL_SHA 0xC001 /**< Weak! */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA 0xC004 -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA 0xC005 - -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_NULL_SHA 0xC006 /**< Weak! */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA 0xC009 -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA 0xC00A - -#define MBEDTLS_TLS_ECDH_RSA_WITH_NULL_SHA 0xC00B /**< Weak! */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA 0xC00E -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA 0xC00F - -#define MBEDTLS_TLS_ECDHE_RSA_WITH_NULL_SHA 0xC010 /**< Weak! */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA 0xC013 -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA 0xC014 - -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 0xC023 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 0xC024 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 0xC025 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 0xC026 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 0xC027 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 0xC028 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 0xC029 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 0xC02A /**< TLS 1.2 */ - -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 0xC02B /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 0xC02C /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 0xC02D /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 0xC02E /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 0xC02F /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 0xC030 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 0xC031 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 0xC032 /**< TLS 1.2 */ - -#define MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA 0xC035 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA 0xC036 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 0xC037 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 0xC038 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA 0xC039 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA256 0xC03A -#define MBEDTLS_TLS_ECDHE_PSK_WITH_NULL_SHA384 0xC03B - -#define MBEDTLS_TLS_RSA_WITH_ARIA_128_CBC_SHA256 0xC03C /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_ARIA_256_CBC_SHA384 0xC03D /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 0xC044 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 0xC045 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 0xC048 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 0xC049 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 0xC04A /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 0xC04B /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 0xC04C /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 0xC04D /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 0xC04E /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 0xC04F /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_ARIA_128_GCM_SHA256 0xC050 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_ARIA_256_GCM_SHA384 0xC051 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 0xC052 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 0xC053 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 0xC05C /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 0xC05D /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 0xC05E /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 0xC05F /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 0xC060 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 0xC061 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 0xC062 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 0xC063 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_ARIA_128_CBC_SHA256 0xC064 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_ARIA_256_CBC_SHA384 0xC065 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 0xC066 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 0xC067 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 0xC068 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 0xC069 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_ARIA_128_GCM_SHA256 0xC06A /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_ARIA_256_GCM_SHA384 0xC06B /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 0xC06C /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 0xC06D /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 0xC06E /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 0xC06F /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 0xC070 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 0xC071 /**< TLS 1.2 */ - -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 0xC072 -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 0xC073 -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 0xC074 -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 0xC075 -#define MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 0xC076 -#define MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 0xC077 -#define MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 0xC078 -#define MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 0xC079 - -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 0xC07A /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 0xC07B /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 0xC07C /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 0xC07D /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 0xC086 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 0xC087 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 0xC088 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 0xC089 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 0xC08A /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 0xC08B /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 0xC08C /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 0xC08D /**< TLS 1.2 */ - -#define MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 0xC08E /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 0xC08F /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 0xC090 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 0xC091 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 0xC092 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 0xC093 /**< TLS 1.2 */ - -#define MBEDTLS_TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 0xC094 -#define MBEDTLS_TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 0xC095 -#define MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 0xC096 -#define MBEDTLS_TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 0xC097 -#define MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 0xC098 -#define MBEDTLS_TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 0xC099 -#define MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 0xC09A -#define MBEDTLS_TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 0xC09B - -#define MBEDTLS_TLS_RSA_WITH_AES_128_CCM 0xC09C /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_AES_256_CCM 0xC09D /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CCM 0xC09E /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CCM 0xC09F /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_AES_128_CCM_8 0xC0A0 /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_WITH_AES_256_CCM_8 0xC0A1 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_128_CCM_8 0xC0A2 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_AES_256_CCM_8 0xC0A3 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_AES_128_CCM 0xC0A4 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_AES_256_CCM 0xC0A5 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CCM 0xC0A6 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CCM 0xC0A7 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_AES_128_CCM_8 0xC0A8 /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_AES_256_CCM_8 0xC0A9 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_128_CCM_8 0xC0AA /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_AES_256_CCM_8 0xC0AB /**< TLS 1.2 */ -/* The last two are named with PSK_DHE in the RFC, which looks like a typo */ - -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CCM 0xC0AC /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CCM 0xC0AD /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 0xC0AE /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 0xC0AF /**< TLS 1.2 */ - -#define MBEDTLS_TLS_ECJPAKE_WITH_AES_128_CCM_8 0xC0FF /**< experimental */ - -/* RFC 7905 */ -#define MBEDTLS_TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 0xCCA8 /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 0xCCA9 /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256 0xCCAA /**< TLS 1.2 */ -#define MBEDTLS_TLS_PSK_WITH_CHACHA20_POLY1305_SHA256 0xCCAB /**< TLS 1.2 */ -#define MBEDTLS_TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 0xCCAC /**< TLS 1.2 */ -#define MBEDTLS_TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256 0xCCAD /**< TLS 1.2 */ -#define MBEDTLS_TLS_RSA_PSK_WITH_CHACHA20_POLY1305_SHA256 0xCCAE /**< TLS 1.2 */ - -/* RFC 8446, Appendix B.4 */ -#define MBEDTLS_TLS1_3_AES_128_GCM_SHA256 0x1301 /**< TLS 1.3 */ -#define MBEDTLS_TLS1_3_AES_256_GCM_SHA384 0x1302 /**< TLS 1.3 */ -#define MBEDTLS_TLS1_3_CHACHA20_POLY1305_SHA256 0x1303 /**< TLS 1.3 */ -#define MBEDTLS_TLS1_3_AES_128_CCM_SHA256 0x1304 /**< TLS 1.3 */ -#define MBEDTLS_TLS1_3_AES_128_CCM_8_SHA256 0x1305 /**< TLS 1.3 */ - -/* Reminder: update mbedtls_ssl_premaster_secret when adding a new key exchange. - * Reminder: update MBEDTLS_KEY_EXCHANGE__xxx below - */ -typedef enum { - MBEDTLS_KEY_EXCHANGE_NONE = 0, - MBEDTLS_KEY_EXCHANGE_RSA, - MBEDTLS_KEY_EXCHANGE_DHE_RSA, - MBEDTLS_KEY_EXCHANGE_ECDHE_RSA, - MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA, - MBEDTLS_KEY_EXCHANGE_PSK, - MBEDTLS_KEY_EXCHANGE_DHE_PSK, - MBEDTLS_KEY_EXCHANGE_RSA_PSK, - MBEDTLS_KEY_EXCHANGE_ECDHE_PSK, - MBEDTLS_KEY_EXCHANGE_ECDH_RSA, - MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA, - MBEDTLS_KEY_EXCHANGE_ECJPAKE, -} mbedtls_key_exchange_type_t; - -/* Key exchanges using a certificate */ -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED -#endif - -/* Key exchanges in either TLS 1.2 or 1.3 which are using an ECDSA - * signature */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_WITH_ECDSA_ANY_ENABLED -#endif - -#if defined(MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED) -#define MBEDTLS_SSL_HANDSHAKE_WITH_CERT_ENABLED -#endif - -/* Key exchanges allowing client certificate requests. - * - * Note: that's almost the same as MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED - * above, except RSA-PSK uses a server certificate but no client cert. - * - * Note: this difference is specific to TLS 1.2, as with TLS 1.3, things are - * more symmetrical: client certs and server certs are either both allowed - * (Ephemeral mode) or both disallowed (PSK and PKS-Ephemeral modes). - */ -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED -#endif - -/* Helper to state that certificate-based client authentication through ECDSA - * is supported in TLS 1.2 */ -#if defined(MBEDTLS_KEY_EXCHANGE_CERT_REQ_ALLOWED_ENABLED) && \ - defined(MBEDTLS_PK_CAN_ECDSA_SIGN) && defined(MBEDTLS_PK_CAN_ECDSA_VERIFY) -#define MBEDTLS_KEY_EXCHANGE_ECDSA_CERT_REQ_ALLOWED_ENABLED -#endif - -/* ECDSA required for certificates in either TLS 1.2 or 1.3 */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECDSA_CERT_REQ_ALLOWED_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_ECDSA_CERT_REQ_ANY_ALLOWED_ENABLED -#endif - -/* Key exchanges involving server signature in ServerKeyExchange */ -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_WITH_SERVER_SIGNATURE_ENABLED -#endif - -/* Key exchanges using ECDH */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_ECDH_ENABLED -#endif - -/* Key exchanges that don't involve ephemeral keys */ -#if defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_NON_PFS_ENABLED -#endif - -/* Key exchanges that involve ephemeral keys */ -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_PFS_ENABLED -#endif - -/* Key exchanges using a PSK */ -#if defined(MBEDTLS_KEY_EXCHANGE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED -#endif - -/* Key exchanges using DHE */ -#if defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_DHE_ENABLED -#endif - -/* Key exchanges using ECDHE */ -#if defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_ECDHE_ENABLED -#endif - -/* TLS 1.2 key exchanges using ECDH or ECDHE*/ -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDHE_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_1_2_ENABLED -#endif - -/* TLS 1.3 PSK key exchanges */ -#if defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED) -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_PSK_ENABLED -#endif - -/* TLS 1.2 or 1.3 key exchanges with PSK */ -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_PSK_ENABLED) -#define MBEDTLS_SSL_HANDSHAKE_WITH_PSK_ENABLED -#endif - -/* TLS 1.3 ephemeral key exchanges */ -#if defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_EPHEMERAL_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_EPHEMERAL_ENABLED) -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_EPHEMERAL_ENABLED -#endif - -/* TLS 1.3 key exchanges using ECDHE */ -#if defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_EPHEMERAL_ENABLED) && \ - defined(PSA_WANT_ALG_ECDH) -#define MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_ECDHE_ENABLED -#endif - -/* TLS 1.2 or 1.3 key exchanges using ECDH or ECDHE */ -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_1_2_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_ECDHE_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_ANY_ENABLED -#endif - -/* TLS 1.2 XXDH key exchanges: ECDH or ECDHE or FFDH */ -#if (defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_1_2_ENABLED) || \ - defined(MBEDTLS_KEY_EXCHANGE_SOME_DHE_ENABLED)) -#define MBEDTLS_KEY_EXCHANGE_SOME_XXDH_1_2_ENABLED -#endif - -/* The handshake params structure has a set of fields called xxdh_psa which are used: - * - by TLS 1.2 with `USE_PSA` to do ECDH or ECDHE; - * - by TLS 1.3 to do ECDHE or FFDHE. - * The following macros can be used to guard their declaration and use. - */ -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_ECDH_OR_ECDHE_1_2_ENABLED) && \ - defined(MBEDTLS_USE_PSA_CRYPTO) -#define MBEDTLS_KEY_EXCHANGE_SOME_XXDH_PSA_1_2_ENABLED -#endif -#if defined(MBEDTLS_KEY_EXCHANGE_SOME_XXDH_PSA_1_2_ENABLED) || \ - defined(MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_SOME_EPHEMERAL_ENABLED) -#define MBEDTLS_KEY_EXCHANGE_SOME_XXDH_PSA_ANY_ENABLED -#endif - -typedef struct mbedtls_ssl_ciphersuite_t mbedtls_ssl_ciphersuite_t; - -#define MBEDTLS_CIPHERSUITE_WEAK 0x01 /**< Weak ciphersuite flag */ -#define MBEDTLS_CIPHERSUITE_SHORT_TAG 0x02 /**< Short authentication tag, - eg for CCM_8 */ -#define MBEDTLS_CIPHERSUITE_NODTLS 0x04 /**< Can't be used with DTLS */ - -/** - * \brief This structure is used for storing ciphersuite information - * - * \note members are defined using integral types instead of enums - * in order to pack structure and reduce memory usage by internal - * \c ciphersuite_definitions[] - */ -struct mbedtls_ssl_ciphersuite_t { - int MBEDTLS_PRIVATE(id); - const char *MBEDTLS_PRIVATE(name); - - uint8_t MBEDTLS_PRIVATE(cipher); /* mbedtls_cipher_type_t */ - uint8_t MBEDTLS_PRIVATE(mac); /* mbedtls_md_type_t */ - uint8_t MBEDTLS_PRIVATE(key_exchange); /* mbedtls_key_exchange_type_t */ - uint8_t MBEDTLS_PRIVATE(flags); - - uint16_t MBEDTLS_PRIVATE(min_tls_version); /* mbedtls_ssl_protocol_version */ - uint16_t MBEDTLS_PRIVATE(max_tls_version); /* mbedtls_ssl_protocol_version */ -}; - -const int *mbedtls_ssl_list_ciphersuites(void); - -const mbedtls_ssl_ciphersuite_t *mbedtls_ssl_ciphersuite_from_string(const char *ciphersuite_name); -const mbedtls_ssl_ciphersuite_t *mbedtls_ssl_ciphersuite_from_id(int ciphersuite_id); - -static inline const char *mbedtls_ssl_ciphersuite_get_name(const mbedtls_ssl_ciphersuite_t *info) -{ - return info->MBEDTLS_PRIVATE(name); -} - -static inline int mbedtls_ssl_ciphersuite_get_id(const mbedtls_ssl_ciphersuite_t *info) -{ - return info->MBEDTLS_PRIVATE(id); -} - -size_t mbedtls_ssl_ciphersuite_get_cipher_key_bitlen(const mbedtls_ssl_ciphersuite_t *info); - -#ifdef __cplusplus -} -#endif - -#endif /* ssl_ciphersuites.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cookie.h b/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cookie.h deleted file mode 100644 index 0f211e632..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_cookie.h +++ /dev/null @@ -1,106 +0,0 @@ -/** - * \file ssl_cookie.h - * - * \brief DTLS cookie callbacks implementation - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SSL_COOKIE_H -#define MBEDTLS_SSL_COOKIE_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/ssl.h" - -#if !defined(MBEDTLS_USE_PSA_CRYPTO) -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif -#endif /* !MBEDTLS_USE_PSA_CRYPTO */ - -/** - * \name SECTION: Module settings - * - * The configuration options you can set for this module are in this section. - * Either change them in mbedtls_config.h or define them on the compiler command line. - * \{ - */ -#ifndef MBEDTLS_SSL_COOKIE_TIMEOUT -#define MBEDTLS_SSL_COOKIE_TIMEOUT 60 /**< Default expiration delay of DTLS cookies, in seconds if HAVE_TIME, or in number of cookies issued */ -#endif - -/** \} name SECTION: Module settings */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \brief Context for the default cookie functions. - */ -typedef struct mbedtls_ssl_cookie_ctx { -#if defined(MBEDTLS_USE_PSA_CRYPTO) - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(psa_hmac_key); /*!< key id for the HMAC portion */ - psa_algorithm_t MBEDTLS_PRIVATE(psa_hmac_alg); /*!< key algorithm for the HMAC portion */ -#else - mbedtls_md_context_t MBEDTLS_PRIVATE(hmac_ctx); /*!< context for the HMAC portion */ -#endif /* MBEDTLS_USE_PSA_CRYPTO */ -#if !defined(MBEDTLS_HAVE_TIME) - unsigned long MBEDTLS_PRIVATE(serial); /*!< serial number for expiration */ -#endif - unsigned long MBEDTLS_PRIVATE(timeout); /*!< timeout delay, in seconds if HAVE_TIME, - or in number of tickets issued */ - -#if !defined(MBEDTLS_USE_PSA_CRYPTO) -#if defined(MBEDTLS_THREADING_C) - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); -#endif -#endif /* !MBEDTLS_USE_PSA_CRYPTO */ -} mbedtls_ssl_cookie_ctx; - -/** - * \brief Initialize cookie context - */ -void mbedtls_ssl_cookie_init(mbedtls_ssl_cookie_ctx *ctx); - -/** - * \brief Setup cookie context (generate keys) - */ -int mbedtls_ssl_cookie_setup(mbedtls_ssl_cookie_ctx *ctx, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -/** - * \brief Set expiration delay for cookies - * (Default MBEDTLS_SSL_COOKIE_TIMEOUT) - * - * \param ctx Cookie context - * \param delay Delay, in seconds if HAVE_TIME, or in number of cookies - * issued in the meantime. - * 0 to disable expiration (NOT recommended) - */ -void mbedtls_ssl_cookie_set_timeout(mbedtls_ssl_cookie_ctx *ctx, unsigned long delay); - -/** - * \brief Free cookie context - */ -void mbedtls_ssl_cookie_free(mbedtls_ssl_cookie_ctx *ctx); - -/** - * \brief Generate cookie, see \c mbedtls_ssl_cookie_write_t - */ -mbedtls_ssl_cookie_write_t mbedtls_ssl_cookie_write; - -/** - * \brief Verify cookie, see \c mbedtls_ssl_cookie_write_t - */ -mbedtls_ssl_cookie_check_t mbedtls_ssl_cookie_check; - -#ifdef __cplusplus -} -#endif - -#endif /* ssl_cookie.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ticket.h b/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ticket.h deleted file mode 100644 index 9f7e44029..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/ssl_ticket.h +++ /dev/null @@ -1,199 +0,0 @@ -/** - * \file ssl_ticket.h - * - * \brief TLS server ticket callbacks implementation - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_SSL_TICKET_H -#define MBEDTLS_SSL_TICKET_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -/* - * This implementation of the session ticket callbacks includes key - * management, rotating the keys periodically in order to preserve forward - * secrecy, when MBEDTLS_HAVE_TIME is defined. - */ - -#include "mbedtls/ssl.h" -#include "mbedtls/cipher.h" - -#if defined(MBEDTLS_HAVE_TIME) -#include "mbedtls/platform_time.h" -#endif - -#if defined(MBEDTLS_USE_PSA_CRYPTO) -#include "psa/crypto.h" -#endif - -#if defined(MBEDTLS_THREADING_C) -#include "mbedtls/threading.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define MBEDTLS_SSL_TICKET_MAX_KEY_BYTES 32 /*!< Max supported key length in bytes */ -#define MBEDTLS_SSL_TICKET_KEY_NAME_BYTES 4 /*!< key name length in bytes */ - -/** - * \brief Information for session ticket protection - */ -typedef struct mbedtls_ssl_ticket_key { - unsigned char MBEDTLS_PRIVATE(name)[MBEDTLS_SSL_TICKET_KEY_NAME_BYTES]; - /*!< random key identifier */ -#if defined(MBEDTLS_HAVE_TIME) - mbedtls_time_t MBEDTLS_PRIVATE(generation_time); /*!< key generation timestamp (seconds) */ -#endif - /*! Lifetime of the key in seconds. This is also the lifetime of the - * tickets created under that key. - */ - uint32_t MBEDTLS_PRIVATE(lifetime); -#if !defined(MBEDTLS_USE_PSA_CRYPTO) - mbedtls_cipher_context_t MBEDTLS_PRIVATE(ctx); /*!< context for auth enc/decryption */ -#else - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(key); /*!< key used for auth enc/decryption */ - psa_algorithm_t MBEDTLS_PRIVATE(alg); /*!< algorithm of auth enc/decryption */ - psa_key_type_t MBEDTLS_PRIVATE(key_type); /*!< key type */ - size_t MBEDTLS_PRIVATE(key_bits); /*!< key length in bits */ -#endif -} -mbedtls_ssl_ticket_key; - -/** - * \brief Context for session ticket handling functions - */ -typedef struct mbedtls_ssl_ticket_context { - mbedtls_ssl_ticket_key MBEDTLS_PRIVATE(keys)[2]; /*!< ticket protection keys */ - unsigned char MBEDTLS_PRIVATE(active); /*!< index of the currently active key */ - - uint32_t MBEDTLS_PRIVATE(ticket_lifetime); /*!< lifetime of tickets in seconds */ - - /** Callback for getting (pseudo-)random numbers */ - int(*MBEDTLS_PRIVATE(f_rng))(void *, unsigned char *, size_t); - void *MBEDTLS_PRIVATE(p_rng); /*!< context for the RNG function */ - -#if defined(MBEDTLS_THREADING_C) - mbedtls_threading_mutex_t MBEDTLS_PRIVATE(mutex); -#endif -} -mbedtls_ssl_ticket_context; - -/** - * \brief Initialize a ticket context. - * (Just make it ready for mbedtls_ssl_ticket_setup() - * or mbedtls_ssl_ticket_free().) - * - * \param ctx Context to be initialized - */ -void mbedtls_ssl_ticket_init(mbedtls_ssl_ticket_context *ctx); - -/** - * \brief Prepare context to be actually used - * - * \param ctx Context to be set up - * \param f_rng RNG callback function (mandatory) - * \param p_rng RNG callback context. - * Note that the RNG callback must remain valid - * until the ticket context is freed. - * \param cipher AEAD cipher to use for ticket protection. - * Recommended value: MBEDTLS_CIPHER_AES_256_GCM. - * \param lifetime Tickets lifetime in seconds - * Recommended value: 86400 (one day). - * - * \note It is highly recommended to select a cipher that is at - * least as strong as the strongest ciphersuite - * supported. Usually that means a 256-bit key. - * - * \note It is recommended to pick a reasonable lifetime so as not - * to negate the benefits of forward secrecy. - * - * \note The TLS 1.3 specification states that ticket lifetime must - * be smaller than seven days. If ticket lifetime has been - * set to a value greater than seven days in this module then - * if the TLS 1.3 is configured to send tickets after the - * handshake it will fail the connection when trying to send - * the first ticket. - * - * \return 0 if successful, - * or a specific MBEDTLS_ERR_XXX error code - */ -int mbedtls_ssl_ticket_setup(mbedtls_ssl_ticket_context *ctx, - mbedtls_f_rng_t *f_rng, void *p_rng, - mbedtls_cipher_type_t cipher, - uint32_t lifetime); - -/** - * \brief Rotate session ticket encryption key to new specified key. - * Provides for external control of session ticket encryption - * key rotation, e.g. for synchronization between different - * machines. If this function is not used, or if not called - * before ticket lifetime expires, then a new session ticket - * encryption key is generated internally in order to avoid - * unbounded session ticket encryption key lifetimes. - * - * \param ctx Context to be set up - * \param name Session ticket encryption key name - * \param nlength Session ticket encryption key name length in bytes - * \param k Session ticket encryption key - * \param klength Session ticket encryption key length in bytes - * \param lifetime Tickets lifetime in seconds - * Recommended value: 86400 (one day). - * - * \note \c name and \c k are recommended to be cryptographically - * random data. - * - * \note \c nlength must match sizeof( ctx->name ) - * - * \note \c klength must be sufficient for use by cipher specified - * to \c mbedtls_ssl_ticket_setup - * - * \note It is recommended to pick a reasonable lifetime so as not - * to negate the benefits of forward secrecy. - * - * \note The TLS 1.3 specification states that ticket lifetime must - * be smaller than seven days. If ticket lifetime has been - * set to a value greater than seven days in this module then - * if the TLS 1.3 is configured to send tickets after the - * handshake it will fail the connection when trying to send - * the first ticket. - * - * \return 0 if successful, - * or a specific MBEDTLS_ERR_XXX error code - */ -int mbedtls_ssl_ticket_rotate(mbedtls_ssl_ticket_context *ctx, - const unsigned char *name, size_t nlength, - const unsigned char *k, size_t klength, - uint32_t lifetime); - -/** - * \brief Implementation of the ticket write callback - * - * \note See \c mbedtls_ssl_ticket_write_t for description - */ -mbedtls_ssl_ticket_write_t mbedtls_ssl_ticket_write; - -/** - * \brief Implementation of the ticket parse callback - * - * \note See \c mbedtls_ssl_ticket_parse_t for description - */ -mbedtls_ssl_ticket_parse_t mbedtls_ssl_ticket_parse; - -/** - * \brief Free a context's content and zeroize it. - * - * \param ctx Context to be cleaned up - */ -void mbedtls_ssl_ticket_free(mbedtls_ssl_ticket_context *ctx); - -#ifdef __cplusplus -} -#endif - -#endif /* ssl_ticket.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/threading.h b/android/mbedtls/arm64-v8a/include/mbedtls/threading.h deleted file mode 100644 index 7b55a4649..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/threading.h +++ /dev/null @@ -1,167 +0,0 @@ -/** - * \file threading.h - * - * \brief Threading abstraction layer - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_THREADING_H -#define MBEDTLS_THREADING_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Bad input parameters to function. */ -#define MBEDTLS_ERR_THREADING_BAD_INPUT_DATA -0x001C -/** Locking / unlocking / free failed with error code. */ -#define MBEDTLS_ERR_THREADING_MUTEX_ERROR -0x001E - -#if defined(MBEDTLS_THREADING_PTHREAD) -#include -typedef struct mbedtls_threading_mutex_t { - pthread_mutex_t MBEDTLS_PRIVATE(mutex); - - /* WARNING - state should only be accessed when holding the mutex lock in - * framework/tests/src/threading_helpers.c, otherwise corruption can occur. - * state will be 0 after a failed init or a free, and nonzero after a - * successful init. This field is for testing only and thus not considered - * part of the public API of Mbed TLS and may change without notice.*/ - char MBEDTLS_PRIVATE(state); - -} mbedtls_threading_mutex_t; -#endif - -#if defined(MBEDTLS_THREADING_ALT) -/* You should define the mbedtls_threading_mutex_t type in your header */ -#include "threading_alt.h" - -/** - * \brief Set your alternate threading implementation function - * pointers and initialize global mutexes. If used, this - * function must be called once in the main thread before any - * other Mbed TLS function is called, and - * mbedtls_threading_free_alt() must be called once in the main - * thread after all other Mbed TLS functions. - * - * \warning \p mutex_init and \p mutex_free don't return a status code. - * If \p mutex_init fails, it should leave the mutex in - * a state such that \p mutex_lock will reliably return - * #MBEDTLS_ERR_THREADING_MUTEX_ERROR called on this mutex, - * and \p mutex_free will do nothing. - * - * \param mutex_init The init function implementation.
- * The behavior is undefined if the mutex is already - * initialized and has not been destroyed. - * On platforms where mutex initialization can fail, - * since this function does not return a status code, - * it must leave the mutex object in a safe state where - * subsequent function calls will not cause undefined - * behavior: after a call to \p mutex_init, the - * function \p mutex_lock must either succeed or - * fail with a nonzero status code, and the function - * \p mutex_free must free any resources associated - * with the mutex.. - * \param mutex_free The destroy function implementation.
- * This function must free any resources associated - * with the mutex object.
- * This function must work reliably if \p mutex_init - * has been called on the mutex and \p mutex_free - * has not yet been called.
- * The behavior is undefined if the mutex was not - * initialized, if it has already been destroyed, - * if it is currently locked, or if this function - * is called concurrently from multiple threads. - * \param mutex_lock The lock function implementation.
- * This function must work reliably on any mutex - * which is not currently locked and on which - * \p mutex_init has already been called but - * \p mutex_free has not been called yet.
- * The behavior is undefined if the mutex was not - * initialized, if it has already been destroyed, or if - * it is currently locked by the calling thread. - * \param mutex_unlock The unlock function implementation.
- * The behavior is undefined if the mutex is not - * currently locked by the calling thread. - */ -void mbedtls_threading_set_alt(void (*mutex_init)(mbedtls_threading_mutex_t *), - void (*mutex_free)(mbedtls_threading_mutex_t *), - int (*mutex_lock)(mbedtls_threading_mutex_t *), - int (*mutex_unlock)(mbedtls_threading_mutex_t *)); - -/** - * \brief Free global mutexes. - */ -void mbedtls_threading_free_alt(void); -#endif /* MBEDTLS_THREADING_ALT */ - -#if defined(MBEDTLS_THREADING_C) -/* - * The function pointers for mutex_init, mutex_free, mutex_ and mutex_unlock - * - * All these functions are expected to work or the result will be undefined. - */ -extern void (*mbedtls_mutex_init)(mbedtls_threading_mutex_t *mutex); -extern void (*mbedtls_mutex_free)(mbedtls_threading_mutex_t *mutex); -extern int (*mbedtls_mutex_lock)(mbedtls_threading_mutex_t *mutex); -extern int (*mbedtls_mutex_unlock)(mbedtls_threading_mutex_t *mutex); - -/* - * Global mutexes - */ -#if defined(MBEDTLS_FS_IO) -extern mbedtls_threading_mutex_t mbedtls_threading_readdir_mutex; -#endif - -#if defined(MBEDTLS_HAVE_TIME_DATE) && !defined(MBEDTLS_PLATFORM_GMTIME_R_ALT) -/* This mutex may or may not be used in the default definition of - * mbedtls_platform_gmtime_r(), but in order to determine that, - * we need to check POSIX features, hence modify _POSIX_C_SOURCE. - * With the current approach, this declaration is orphaned, lacking - * an accompanying definition, in case mbedtls_platform_gmtime_r() - * doesn't need it, but that's not a problem. */ -extern mbedtls_threading_mutex_t mbedtls_threading_gmtime_mutex; -#endif /* MBEDTLS_HAVE_TIME_DATE && !MBEDTLS_PLATFORM_GMTIME_R_ALT */ - -#if defined(MBEDTLS_PSA_CRYPTO_C) -/* - * A mutex used to make the PSA subsystem thread safe. - * - * key_slot_mutex protects the registered_readers and - * state variable for all key slots in &global_data.key_slots. - * - * This mutex must be held when any read from or write to a state or - * registered_readers field is performed, i.e. when calling functions: - * psa_key_slot_state_transition(), psa_register_read(), psa_unregister_read(), - * psa_key_slot_has_readers() and psa_wipe_key_slot(). */ -extern mbedtls_threading_mutex_t mbedtls_threading_key_slot_mutex; - -/* - * A mutex used to make the non-rng PSA global_data struct members thread safe. - * - * This mutex must be held when reading or writing to any of the PSA global_data - * structure members, other than the rng_state or rng struct. */ -extern mbedtls_threading_mutex_t mbedtls_threading_psa_globaldata_mutex; - -/* - * A mutex used to make the PSA global_data rng data thread safe. - * - * This mutex must be held when reading or writing to the PSA - * global_data rng_state or rng struct members. */ -extern mbedtls_threading_mutex_t mbedtls_threading_psa_rngdata_mutex; -#endif - -#endif /* MBEDTLS_THREADING_C */ - -#ifdef __cplusplus -} -#endif - -#endif /* threading.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/timing.h b/android/mbedtls/arm64-v8a/include/mbedtls/timing.h deleted file mode 100644 index 62ae1022d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/timing.h +++ /dev/null @@ -1,94 +0,0 @@ -/** - * \file timing.h - * - * \brief Portable interface to timeouts and to the CPU cycle counter - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_TIMING_H -#define MBEDTLS_TIMING_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#if !defined(MBEDTLS_TIMING_ALT) -// Regular implementation -// - -/** - * \brief timer structure - */ -struct mbedtls_timing_hr_time { - uint64_t MBEDTLS_PRIVATE(opaque)[4]; -}; - -/** - * \brief Context for mbedtls_timing_set/get_delay() - */ -typedef struct mbedtls_timing_delay_context { - struct mbedtls_timing_hr_time MBEDTLS_PRIVATE(timer); - uint32_t MBEDTLS_PRIVATE(int_ms); - uint32_t MBEDTLS_PRIVATE(fin_ms); -} mbedtls_timing_delay_context; - -#else /* MBEDTLS_TIMING_ALT */ -#include "timing_alt.h" -#endif /* MBEDTLS_TIMING_ALT */ - -/* Internal use */ -unsigned long mbedtls_timing_get_timer(struct mbedtls_timing_hr_time *val, int reset); - -/** - * \brief Set a pair of delays to watch - * (See \c mbedtls_timing_get_delay().) - * - * \param data Pointer to timing data. - * Must point to a valid \c mbedtls_timing_delay_context struct. - * \param int_ms First (intermediate) delay in milliseconds. - * The effect if int_ms > fin_ms is unspecified. - * \param fin_ms Second (final) delay in milliseconds. - * Pass 0 to cancel the current delay. - * - * \note To set a single delay, either use \c mbedtls_timing_set_timer - * directly or use this function with int_ms == fin_ms. - */ -void mbedtls_timing_set_delay(void *data, uint32_t int_ms, uint32_t fin_ms); - -/** - * \brief Get the status of delays - * (Memory helper: number of delays passed.) - * - * \param data Pointer to timing data - * Must point to a valid \c mbedtls_timing_delay_context struct. - * - * \return -1 if cancelled (fin_ms = 0), - * 0 if none of the delays are passed, - * 1 if only the intermediate delay is passed, - * 2 if the final delay is passed. - */ -int mbedtls_timing_get_delay(void *data); - -/** - * \brief Get the final timing delay - * - * \param data Pointer to timing data - * Must point to a valid \c mbedtls_timing_delay_context struct. - * - * \return Final timing delay in milliseconds. - */ -uint32_t mbedtls_timing_get_final_delay( - const mbedtls_timing_delay_context *data); - -#ifdef __cplusplus -} -#endif - -#endif /* timing.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/version.h b/android/mbedtls/arm64-v8a/include/mbedtls/version.h deleted file mode 100644 index 637f9d38b..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/version.h +++ /dev/null @@ -1,78 +0,0 @@ -/** - * \file version.h - * - * \brief Run-time version information - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -/* - * This set of run-time variables can be used to determine the version number of - * the Mbed TLS library used. Compile-time version defines for the same can be - * found in build_info.h - */ -#ifndef MBEDTLS_VERSION_H -#define MBEDTLS_VERSION_H - -#include "mbedtls/build_info.h" - -#if defined(MBEDTLS_VERSION_C) - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Get the version number. - * - * \return The constructed version number in the format - * MMNNPP00 (Major, Minor, Patch). - */ -unsigned int mbedtls_version_get_number(void); - -/** - * Get the version string ("x.y.z"). - * - * \param string The string that will receive the value. - * (Should be at least 9 bytes in size) - */ -void mbedtls_version_get_string(char *string); - -/** - * Get the full version string ("Mbed TLS x.y.z"). - * - * \param string The string that will receive the value. The Mbed TLS version - * string will use 18 bytes AT MOST including a terminating - * null byte. - * (So the buffer should be at least 18 bytes to receive this - * version string). - */ -void mbedtls_version_get_string_full(char *string); - -/** - * \brief Check if support for a feature was compiled into this - * Mbed TLS binary. This allows you to see at runtime if the - * library was for instance compiled with or without - * Multi-threading support. - * - * \note only checks against defines in the sections "System - * support", "Mbed TLS modules" and "Mbed TLS feature - * support" in mbedtls_config.h - * - * \param feature The string for the define to check (e.g. "MBEDTLS_AES_C") - * - * \return 0 if the feature is present, - * -1 if the feature is not present and - * -2 if support for feature checking as a whole was not - * compiled in. - */ -int mbedtls_version_check_feature(const char *feature); - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_VERSION_C */ - -#endif /* version.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/x509.h b/android/mbedtls/arm64-v8a/include/mbedtls/x509.h deleted file mode 100644 index 6b104613d..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/x509.h +++ /dev/null @@ -1,500 +0,0 @@ -/** - * \file x509.h - * - * \brief X.509 generic defines and structures - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_X509_H -#define MBEDTLS_X509_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/asn1.h" -#include "mbedtls/pk.h" - -#if defined(MBEDTLS_RSA_C) -#include "mbedtls/rsa.h" -#endif - -/** - * \addtogroup x509_module - * \{ - */ - -#if !defined(MBEDTLS_X509_MAX_INTERMEDIATE_CA) -/** - * Maximum number of intermediate CAs in a verification chain. - * That is, maximum length of the chain, excluding the end-entity certificate - * and the trusted root certificate. - * - * Set this to a low value to prevent an adversary from making you waste - * resources verifying an overlong certificate chain. - */ -#define MBEDTLS_X509_MAX_INTERMEDIATE_CA 8 -#endif - -/** - * \name X509 Error codes - * \{ - */ -/** Unavailable feature, e.g. RSA hashing/encryption combination. */ -#define MBEDTLS_ERR_X509_FEATURE_UNAVAILABLE -0x2080 -/** Requested OID is unknown. */ -#define MBEDTLS_ERR_X509_UNKNOWN_OID -0x2100 -/** The CRT/CRL/CSR format is invalid, e.g. different type expected. */ -#define MBEDTLS_ERR_X509_INVALID_FORMAT -0x2180 -/** The CRT/CRL/CSR version element is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_VERSION -0x2200 -/** The serial tag or value is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_SERIAL -0x2280 -/** The algorithm tag or value is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_ALG -0x2300 -/** The name tag or value is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_NAME -0x2380 -/** The date tag or value is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_DATE -0x2400 -/** The signature tag or value invalid. */ -#define MBEDTLS_ERR_X509_INVALID_SIGNATURE -0x2480 -/** The extension tag or value is invalid. */ -#define MBEDTLS_ERR_X509_INVALID_EXTENSIONS -0x2500 -/** CRT/CRL/CSR has an unsupported version number. */ -#define MBEDTLS_ERR_X509_UNKNOWN_VERSION -0x2580 -/** Signature algorithm (oid) is unsupported. */ -#define MBEDTLS_ERR_X509_UNKNOWN_SIG_ALG -0x2600 -/** Signature algorithms do not match. (see \c ::mbedtls_x509_crt sig_oid) */ -#define MBEDTLS_ERR_X509_SIG_MISMATCH -0x2680 -/** Certificate verification failed, e.g. CRL, CA or signature check failed. */ -#define MBEDTLS_ERR_X509_CERT_VERIFY_FAILED -0x2700 -/** Format not recognized as DER or PEM. */ -#define MBEDTLS_ERR_X509_CERT_UNKNOWN_FORMAT -0x2780 -/** Input invalid. */ -#define MBEDTLS_ERR_X509_BAD_INPUT_DATA -0x2800 -/** Allocation of memory failed. */ -#define MBEDTLS_ERR_X509_ALLOC_FAILED -0x2880 -/** Read/write of file failed. */ -#define MBEDTLS_ERR_X509_FILE_IO_ERROR -0x2900 -/** Destination buffer is too small. */ -#define MBEDTLS_ERR_X509_BUFFER_TOO_SMALL -0x2980 -/** A fatal error occurred, eg the chain is too long or the vrfy callback failed. */ -#define MBEDTLS_ERR_X509_FATAL_ERROR -0x3000 -/** \} name X509 Error codes */ - -/** - * \name X509 Verify codes - * \{ - */ -/* Reminder: update x509_crt_verify_strings[] in library/x509_crt.c */ -#define MBEDTLS_X509_BADCERT_EXPIRED 0x01 /**< The certificate validity has expired. */ -#define MBEDTLS_X509_BADCERT_REVOKED 0x02 /**< The certificate has been revoked (is on a CRL). */ -#define MBEDTLS_X509_BADCERT_CN_MISMATCH 0x04 /**< The certificate Common Name (CN) does not match with the expected CN. */ -#define MBEDTLS_X509_BADCERT_NOT_TRUSTED 0x08 /**< The certificate is not correctly signed by the trusted CA. */ -#define MBEDTLS_X509_BADCRL_NOT_TRUSTED 0x10 /**< The CRL is not correctly signed by the trusted CA. */ -#define MBEDTLS_X509_BADCRL_EXPIRED 0x20 /**< The CRL is expired. */ -#define MBEDTLS_X509_BADCERT_MISSING 0x40 /**< Certificate was missing. */ -#define MBEDTLS_X509_BADCERT_SKIP_VERIFY 0x80 /**< Certificate verification was skipped. */ -#define MBEDTLS_X509_BADCERT_OTHER 0x0100 /**< Other reason (can be used by verify callback) */ -#define MBEDTLS_X509_BADCERT_FUTURE 0x0200 /**< The certificate validity starts in the future. */ -#define MBEDTLS_X509_BADCRL_FUTURE 0x0400 /**< The CRL is from the future */ -#define MBEDTLS_X509_BADCERT_KEY_USAGE 0x0800 /**< Usage does not match the keyUsage extension. */ -#define MBEDTLS_X509_BADCERT_EXT_KEY_USAGE 0x1000 /**< Usage does not match the extendedKeyUsage extension. */ -#define MBEDTLS_X509_BADCERT_NS_CERT_TYPE 0x2000 /**< Usage does not match the nsCertType extension. */ -#define MBEDTLS_X509_BADCERT_BAD_MD 0x4000 /**< The certificate is signed with an unacceptable hash. */ -#define MBEDTLS_X509_BADCERT_BAD_PK 0x8000 /**< The certificate is signed with an unacceptable PK alg (eg RSA vs ECDSA). */ -#define MBEDTLS_X509_BADCERT_BAD_KEY 0x010000 /**< The certificate is signed with an unacceptable key (eg bad curve, RSA too short). */ -#define MBEDTLS_X509_BADCRL_BAD_MD 0x020000 /**< The CRL is signed with an unacceptable hash. */ -#define MBEDTLS_X509_BADCRL_BAD_PK 0x040000 /**< The CRL is signed with an unacceptable PK alg (eg RSA vs ECDSA). */ -#define MBEDTLS_X509_BADCRL_BAD_KEY 0x080000 /**< The CRL is signed with an unacceptable key (eg bad curve, RSA too short). */ - -/** \} name X509 Verify codes */ -/** \} addtogroup x509_module */ - -/* - * X.509 v3 Subject Alternative Name types. - * otherName [0] OtherName, - * rfc822Name [1] IA5String, - * dNSName [2] IA5String, - * x400Address [3] ORAddress, - * directoryName [4] Name, - * ediPartyName [5] EDIPartyName, - * uniformResourceIdentifier [6] IA5String, - * iPAddress [7] OCTET STRING, - * registeredID [8] OBJECT IDENTIFIER - */ -#define MBEDTLS_X509_SAN_OTHER_NAME 0 -#define MBEDTLS_X509_SAN_RFC822_NAME 1 -#define MBEDTLS_X509_SAN_DNS_NAME 2 -#define MBEDTLS_X509_SAN_X400_ADDRESS_NAME 3 -#define MBEDTLS_X509_SAN_DIRECTORY_NAME 4 -#define MBEDTLS_X509_SAN_EDI_PARTY_NAME 5 -#define MBEDTLS_X509_SAN_UNIFORM_RESOURCE_IDENTIFIER 6 -#define MBEDTLS_X509_SAN_IP_ADDRESS 7 -#define MBEDTLS_X509_SAN_REGISTERED_ID 8 - -/* - * X.509 v3 Key Usage Extension flags - * Reminder: update mbedtls_x509_info_key_usage() when adding new flags. - */ -#define MBEDTLS_X509_KU_DIGITAL_SIGNATURE (0x80) /* bit 0 */ -#define MBEDTLS_X509_KU_NON_REPUDIATION (0x40) /* bit 1 */ -#define MBEDTLS_X509_KU_KEY_ENCIPHERMENT (0x20) /* bit 2 */ -#define MBEDTLS_X509_KU_DATA_ENCIPHERMENT (0x10) /* bit 3 */ -#define MBEDTLS_X509_KU_KEY_AGREEMENT (0x08) /* bit 4 */ -#define MBEDTLS_X509_KU_KEY_CERT_SIGN (0x04) /* bit 5 */ -#define MBEDTLS_X509_KU_CRL_SIGN (0x02) /* bit 6 */ -#define MBEDTLS_X509_KU_ENCIPHER_ONLY (0x01) /* bit 7 */ -#define MBEDTLS_X509_KU_DECIPHER_ONLY (0x8000) /* bit 8 */ - -/* - * Netscape certificate types - * (http://www.mozilla.org/projects/security/pki/nss/tech-notes/tn3.html) - */ - -#define MBEDTLS_X509_NS_CERT_TYPE_SSL_CLIENT (0x80) /* bit 0 */ -#define MBEDTLS_X509_NS_CERT_TYPE_SSL_SERVER (0x40) /* bit 1 */ -#define MBEDTLS_X509_NS_CERT_TYPE_EMAIL (0x20) /* bit 2 */ -#define MBEDTLS_X509_NS_CERT_TYPE_OBJECT_SIGNING (0x10) /* bit 3 */ -#define MBEDTLS_X509_NS_CERT_TYPE_RESERVED (0x08) /* bit 4 */ -#define MBEDTLS_X509_NS_CERT_TYPE_SSL_CA (0x04) /* bit 5 */ -#define MBEDTLS_X509_NS_CERT_TYPE_EMAIL_CA (0x02) /* bit 6 */ -#define MBEDTLS_X509_NS_CERT_TYPE_OBJECT_SIGNING_CA (0x01) /* bit 7 */ - -/* - * X.509 extension types - * - * Comments refer to the status for using certificates. Status can be - * different for writing certificates or reading CRLs or CSRs. - * - * Those are defined in oid.h as oid.c needs them in a data structure. Since - * these were previously defined here, let's have aliases for compatibility. - */ -#define MBEDTLS_X509_EXT_AUTHORITY_KEY_IDENTIFIER MBEDTLS_OID_X509_EXT_AUTHORITY_KEY_IDENTIFIER -#define MBEDTLS_X509_EXT_SUBJECT_KEY_IDENTIFIER MBEDTLS_OID_X509_EXT_SUBJECT_KEY_IDENTIFIER -#define MBEDTLS_X509_EXT_KEY_USAGE MBEDTLS_OID_X509_EXT_KEY_USAGE -#define MBEDTLS_X509_EXT_CERTIFICATE_POLICIES MBEDTLS_OID_X509_EXT_CERTIFICATE_POLICIES -#define MBEDTLS_X509_EXT_POLICY_MAPPINGS MBEDTLS_OID_X509_EXT_POLICY_MAPPINGS -#define MBEDTLS_X509_EXT_SUBJECT_ALT_NAME MBEDTLS_OID_X509_EXT_SUBJECT_ALT_NAME /* Supported (DNS) */ -#define MBEDTLS_X509_EXT_ISSUER_ALT_NAME MBEDTLS_OID_X509_EXT_ISSUER_ALT_NAME -#define MBEDTLS_X509_EXT_SUBJECT_DIRECTORY_ATTRS MBEDTLS_OID_X509_EXT_SUBJECT_DIRECTORY_ATTRS -#define MBEDTLS_X509_EXT_BASIC_CONSTRAINTS MBEDTLS_OID_X509_EXT_BASIC_CONSTRAINTS /* Supported */ -#define MBEDTLS_X509_EXT_NAME_CONSTRAINTS MBEDTLS_OID_X509_EXT_NAME_CONSTRAINTS -#define MBEDTLS_X509_EXT_POLICY_CONSTRAINTS MBEDTLS_OID_X509_EXT_POLICY_CONSTRAINTS -#define MBEDTLS_X509_EXT_EXTENDED_KEY_USAGE MBEDTLS_OID_X509_EXT_EXTENDED_KEY_USAGE -#define MBEDTLS_X509_EXT_CRL_DISTRIBUTION_POINTS MBEDTLS_OID_X509_EXT_CRL_DISTRIBUTION_POINTS -#define MBEDTLS_X509_EXT_INIHIBIT_ANYPOLICY MBEDTLS_OID_X509_EXT_INIHIBIT_ANYPOLICY -#define MBEDTLS_X509_EXT_FRESHEST_CRL MBEDTLS_OID_X509_EXT_FRESHEST_CRL -#define MBEDTLS_X509_EXT_NS_CERT_TYPE MBEDTLS_OID_X509_EXT_NS_CERT_TYPE - -/* - * Storage format identifiers - * Recognized formats: PEM and DER - */ -#define MBEDTLS_X509_FORMAT_DER 1 -#define MBEDTLS_X509_FORMAT_PEM 2 - -#define MBEDTLS_X509_MAX_DN_NAME_SIZE 256 /**< Maximum value size of a DN entry */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \addtogroup x509_module - * \{ */ - -/** - * \name Structures for parsing X.509 certificates, CRLs and CSRs - * \{ - */ - -/** - * Type-length-value structure that allows for ASN1 using DER. - */ -typedef mbedtls_asn1_buf mbedtls_x509_buf; - -/** - * Container for ASN1 bit strings. - */ -typedef mbedtls_asn1_bitstring mbedtls_x509_bitstring; - -/** - * Container for ASN1 named information objects. - * It allows for Relative Distinguished Names (e.g. cn=localhost,ou=code,etc.). - */ -typedef mbedtls_asn1_named_data mbedtls_x509_name; - -/** - * Container for a sequence of ASN.1 items - */ -typedef mbedtls_asn1_sequence mbedtls_x509_sequence; - -/* - * Container for the fields of the Authority Key Identifier object - */ -typedef struct mbedtls_x509_authority { - mbedtls_x509_buf keyIdentifier; - mbedtls_x509_sequence authorityCertIssuer; - mbedtls_x509_buf authorityCertSerialNumber; - mbedtls_x509_buf raw; -} -mbedtls_x509_authority; - -/** Container for date and time (precision in seconds). */ -typedef struct mbedtls_x509_time { - int year, mon, day; /**< Date. */ - int hour, min, sec; /**< Time. */ -} -mbedtls_x509_time; - -/** - * From RFC 5280 section 4.2.1.6: - * OtherName ::= SEQUENCE { - * type-id OBJECT IDENTIFIER, - * value [0] EXPLICIT ANY DEFINED BY type-id } - * - * Future versions of the library may add new fields to this structure or - * to its embedded union and structure. - */ -typedef struct mbedtls_x509_san_other_name { - /** - * The type_id is an OID as defined in RFC 5280. - * To check the value of the type id, you should use - * \p MBEDTLS_OID_CMP with a known OID mbedtls_x509_buf. - */ - mbedtls_x509_buf type_id; /**< The type id. */ - union { - /** - * From RFC 4108 section 5: - * HardwareModuleName ::= SEQUENCE { - * hwType OBJECT IDENTIFIER, - * hwSerialNum OCTET STRING } - */ - struct { - mbedtls_x509_buf oid; /**< The object identifier. */ - mbedtls_x509_buf val; /**< The named value. */ - } - hardware_module_name; - } - value; -} -mbedtls_x509_san_other_name; - -/** - * A structure for holding the parsed Subject Alternative Name, - * according to type. - * - * Future versions of the library may add new fields to this structure or - * to its embedded union and structure. - */ -typedef struct mbedtls_x509_subject_alternative_name { - int type; /**< The SAN type, value of MBEDTLS_X509_SAN_XXX. */ - union { - mbedtls_x509_san_other_name other_name; - mbedtls_x509_name directory_name; - mbedtls_x509_buf unstructured_name; /**< The buffer for the unstructured types. rfc822Name, dnsName and uniformResourceIdentifier are currently supported. */ - } - san; /**< A union of the supported SAN types */ -} -mbedtls_x509_subject_alternative_name; - -typedef struct mbedtls_x509_san_list { - mbedtls_x509_subject_alternative_name node; - struct mbedtls_x509_san_list *next; -} -mbedtls_x509_san_list; - -/** \} name Structures for parsing X.509 certificates, CRLs and CSRs */ -/** \} addtogroup x509_module */ - -/** - * \brief Store the certificate DN in printable form into buf; - * no more than size characters will be written. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param dn The X509 name to represent - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_dn_gets(char *buf, size_t size, const mbedtls_x509_name *dn); - -/** - * \brief Convert the certificate DN string \p name into - * a linked list of mbedtls_x509_name (equivalent to - * mbedtls_asn1_named_data). - * - * \note This function allocates a linked list, and places the head - * pointer in \p head. This list must later be freed by a - * call to mbedtls_asn1_free_named_data_list(). - * - * \param[out] head Address in which to store the pointer to the head of the - * allocated list of mbedtls_x509_name. Must point to NULL on - * entry. - * \param[in] name The string representation of a DN to convert - * - * \return 0 on success, or a negative error code. - */ -int mbedtls_x509_string_to_names(mbedtls_asn1_named_data **head, const char *name); - -/** - * \brief Return the next relative DN in an X509 name. - * - * \note Intended use is to compare function result to dn->next - * in order to detect boundaries of multi-valued RDNs. - * - * \param dn Current node in the X509 name - * - * \return Pointer to the first attribute-value pair of the - * next RDN in sequence, or NULL if end is reached. - */ -static inline mbedtls_x509_name *mbedtls_x509_dn_get_next( - mbedtls_x509_name *dn) -{ - while (dn->MBEDTLS_PRIVATE(next_merged) && dn->next != NULL) { - dn = dn->next; - } - return dn->next; -} - -/** - * \brief Store the certificate serial in printable form into buf; - * no more than size characters will be written. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param serial The X509 serial to represent - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_serial_gets(char *buf, size_t size, const mbedtls_x509_buf *serial); - -/** - * \brief Compare pair of mbedtls_x509_time. - * - * \param t1 mbedtls_x509_time to compare - * \param t2 mbedtls_x509_time to compare - * - * \return < 0 if t1 is before t2 - * 0 if t1 equals t2 - * > 0 if t1 is after t2 - */ -int mbedtls_x509_time_cmp(const mbedtls_x509_time *t1, const mbedtls_x509_time *t2); - -#if defined(MBEDTLS_HAVE_TIME_DATE) -/** - * \brief Fill mbedtls_x509_time with provided mbedtls_time_t. - * - * \param tt mbedtls_time_t to convert - * \param now mbedtls_x509_time to fill with converted mbedtls_time_t - * - * \return \c 0 on success - * \return A non-zero return value on failure. - */ -int mbedtls_x509_time_gmtime(mbedtls_time_t tt, mbedtls_x509_time *now); -#endif /* MBEDTLS_HAVE_TIME_DATE */ - -/** - * \brief Check a given mbedtls_x509_time against the system time - * and tell if it's in the past. - * - * \note Intended usage is "if( is_past( valid_to ) ) ERROR". - * Hence the return value of 1 if on internal errors. - * - * \param to mbedtls_x509_time to check - * - * \return 1 if the given time is in the past or an error occurred, - * 0 otherwise. - */ -int mbedtls_x509_time_is_past(const mbedtls_x509_time *to); - -/** - * \brief Check a given mbedtls_x509_time against the system time - * and tell if it's in the future. - * - * \note Intended usage is "if( is_future( valid_from ) ) ERROR". - * Hence the return value of 1 if on internal errors. - * - * \param from mbedtls_x509_time to check - * - * \return 1 if the given time is in the future or an error occurred, - * 0 otherwise. - */ -int mbedtls_x509_time_is_future(const mbedtls_x509_time *from); - -/** - * \brief This function parses an item in the SubjectAlternativeNames - * extension. Please note that this function might allocate - * additional memory for a subject alternative name, thus - * mbedtls_x509_free_subject_alt_name has to be called - * to dispose of this additional memory afterwards. - * - * \param san_buf The buffer holding the raw data item of the subject - * alternative name. - * \param san The target structure to populate with the parsed presentation - * of the subject alternative name encoded in \p san_buf. - * - * \note Supported GeneralName types, as defined in RFC 5280: - * "rfc822Name", "dnsName", "directoryName", - * "uniformResourceIdentifier" and "hardware_module_name" - * of type "otherName", as defined in RFC 4108. - * - * \note This function should be called on a single raw data of - * subject alternative name. For example, after successful - * certificate parsing, one must iterate on every item in the - * \c crt->subject_alt_names sequence, and pass it to - * this function. - * - * \warning The target structure contains pointers to the raw data of the - * parsed certificate, and its lifetime is restricted by the - * lifetime of the certificate. - * - * \return \c 0 on success - * \return #MBEDTLS_ERR_X509_FEATURE_UNAVAILABLE for an unsupported - * SAN type. - * \return Another negative value for any other failure. - */ -int mbedtls_x509_parse_subject_alt_name(const mbedtls_x509_buf *san_buf, - mbedtls_x509_subject_alternative_name *san); -/** - * \brief Unallocate all data related to subject alternative name - * - * \param san SAN structure - extra memory owned by this structure will be freed - */ -void mbedtls_x509_free_subject_alt_name(mbedtls_x509_subject_alternative_name *san); - -/** - * \brief This function parses a CN string as an IP address. - * - * \param cn The CN string to parse. CN string MUST be null-terminated. - * \param dst The target buffer to populate with the binary IP address. - * The buffer MUST be 16 bytes to save IPv6, and should be - * 4-byte aligned if the result will be used as struct in_addr. - * e.g. uint32_t dst[4] - * - * \note \p cn is parsed as an IPv6 address if string contains ':', - * else \p cn is parsed as an IPv4 address. - * - * \return Length of binary IP address; num bytes written to target. - * \return \c 0 on failure to parse CN string as an IP address. - */ -size_t mbedtls_x509_crt_parse_cn_inet_pton(const char *cn, void *dst); - -#define MBEDTLS_X509_SAFE_SNPRINTF \ - do { \ - if (ret < 0 || (size_t) ret >= n) \ - return MBEDTLS_ERR_X509_BUFFER_TOO_SMALL; \ - \ - n -= (size_t) ret; \ - p += (size_t) ret; \ - } while (0) - -#ifdef __cplusplus -} -#endif - -#endif /* MBEDTLS_X509_H */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/x509_crl.h b/android/mbedtls/arm64-v8a/include/mbedtls/x509_crl.h deleted file mode 100644 index 6625a44f4..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/x509_crl.h +++ /dev/null @@ -1,184 +0,0 @@ -/** - * \file x509_crl.h - * - * \brief X.509 certificate revocation list parsing - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_X509_CRL_H -#define MBEDTLS_X509_CRL_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/x509.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \addtogroup x509_module - * \{ */ - -/** - * \name Structures and functions for parsing CRLs - * \{ - */ - -/** - * Certificate revocation list entry. - * Contains the CA-specific serial numbers and revocation dates. - * - * Some fields of this structure are publicly readable. Do not modify - * them except via Mbed TLS library functions: the effect of modifying - * those fields or the data that those fields points to is unspecified. - */ -typedef struct mbedtls_x509_crl_entry { - /** Direct access to the whole entry inside the containing buffer. */ - mbedtls_x509_buf raw; - /** The serial number of the revoked certificate. */ - mbedtls_x509_buf serial; - /** The revocation date of this entry. */ - mbedtls_x509_time revocation_date; - /** Direct access to the list of CRL entry extensions - * (an ASN.1 constructed sequence). - * - * If there are no extensions, `entry_ext.len == 0` and - * `entry_ext.p == NULL`. */ - mbedtls_x509_buf entry_ext; - - /** Next element in the linked list of entries. - * \p NULL indicates the end of the list. - * Do not modify this field directly. */ - struct mbedtls_x509_crl_entry *next; -} -mbedtls_x509_crl_entry; - -/** - * Certificate revocation list structure. - * Every CRL may have multiple entries. - */ -typedef struct mbedtls_x509_crl { - mbedtls_x509_buf raw; /**< The raw certificate data (DER). */ - mbedtls_x509_buf tbs; /**< The raw certificate body (DER). The part that is To Be Signed. */ - - int version; /**< CRL version (1=v1, 2=v2) */ - mbedtls_x509_buf sig_oid; /**< CRL signature type identifier */ - - mbedtls_x509_buf issuer_raw; /**< The raw issuer data (DER). */ - - mbedtls_x509_name issuer; /**< The parsed issuer data (named information object). */ - - mbedtls_x509_time this_update; - mbedtls_x509_time next_update; - - mbedtls_x509_crl_entry entry; /**< The CRL entries containing the certificate revocation times for this CA. */ - - mbedtls_x509_buf crl_ext; - - mbedtls_x509_buf MBEDTLS_PRIVATE(sig_oid2); - mbedtls_x509_buf MBEDTLS_PRIVATE(sig); - mbedtls_md_type_t MBEDTLS_PRIVATE(sig_md); /**< Internal representation of the MD algorithm of the signature algorithm, e.g. MBEDTLS_MD_SHA256 */ - mbedtls_pk_type_t MBEDTLS_PRIVATE(sig_pk); /**< Internal representation of the Public Key algorithm of the signature algorithm, e.g. MBEDTLS_PK_RSA */ - void *MBEDTLS_PRIVATE(sig_opts); /**< Signature options to be passed to mbedtls_pk_verify_ext(), e.g. for RSASSA-PSS */ - - /** Next element in the linked list of CRL. - * \p NULL indicates the end of the list. - * Do not modify this field directly. */ - struct mbedtls_x509_crl *next; -} -mbedtls_x509_crl; - -/** - * \brief Parse a DER-encoded CRL and append it to the chained list - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain points to the start of the chain - * \param buf buffer holding the CRL data in DER format - * \param buflen size of the buffer - * (including the terminating null byte for PEM data) - * - * \return 0 if successful, or a specific X509 or PEM error code - */ -int mbedtls_x509_crl_parse_der(mbedtls_x509_crl *chain, - const unsigned char *buf, size_t buflen); -/** - * \brief Parse one or more CRLs and append them to the chained list - * - * \note Multiple CRLs are accepted only if using PEM format - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain points to the start of the chain - * \param buf buffer holding the CRL data in PEM or DER format - * \param buflen size of the buffer - * (including the terminating null byte for PEM data) - * - * \return 0 if successful, or a specific X509 or PEM error code - */ -int mbedtls_x509_crl_parse(mbedtls_x509_crl *chain, const unsigned char *buf, size_t buflen); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief Load one or more CRLs and append them to the chained list - * - * \note Multiple CRLs are accepted only if using PEM format - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain points to the start of the chain - * \param path filename to read the CRLs from (in PEM or DER encoding) - * - * \return 0 if successful, or a specific X509 or PEM error code - */ -int mbedtls_x509_crl_parse_file(mbedtls_x509_crl *chain, const char *path); -#endif /* MBEDTLS_FS_IO */ - -#if !defined(MBEDTLS_X509_REMOVE_INFO) -/** - * \brief Returns an informational string about the CRL. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param prefix A line prefix - * \param crl The X509 CRL to represent - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_crl_info(char *buf, size_t size, const char *prefix, - const mbedtls_x509_crl *crl); -#endif /* !MBEDTLS_X509_REMOVE_INFO */ - -/** - * \brief Initialize a CRL (chain) - * - * \param crl CRL chain to initialize - */ -void mbedtls_x509_crl_init(mbedtls_x509_crl *crl); - -/** - * \brief Unallocate all CRL data - * - * \param crl CRL chain to free - */ -void mbedtls_x509_crl_free(mbedtls_x509_crl *crl); - -/** \} name Structures and functions for parsing CRLs */ -/** \} addtogroup x509_module */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_x509_crl.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/x509_crt.h b/android/mbedtls/arm64-v8a/include/mbedtls/x509_crt.h deleted file mode 100644 index 6b9603959..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/x509_crt.h +++ /dev/null @@ -1,1208 +0,0 @@ -/** - * \file x509_crt.h - * - * \brief X.509 certificate parsing and writing - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_X509_CRT_H -#define MBEDTLS_X509_CRT_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/x509.h" -#include "mbedtls/x509_crl.h" -#include "mbedtls/bignum.h" - -/** - * \addtogroup x509_module - * \{ - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \name Structures and functions for parsing and writing X.509 certificates - * \{ - */ - -/** - * Container for an X.509 certificate. The certificate may be chained. - * - * Some fields of this structure are publicly readable. Do not modify - * them except via Mbed TLS library functions: the effect of modifying - * those fields or the data that those fields points to is unspecified. - */ -typedef struct mbedtls_x509_crt { - int MBEDTLS_PRIVATE(own_buffer); /**< Indicates if \c raw is owned - * by the structure or not. */ - mbedtls_x509_buf raw; /**< The raw certificate data (DER). */ - mbedtls_x509_buf tbs; /**< The raw certificate body (DER). The part that is To Be Signed. */ - - int version; /**< The X.509 version. (1=v1, 2=v2, 3=v3) */ - mbedtls_x509_buf serial; /**< Unique id for certificate issued by a specific CA. */ - mbedtls_x509_buf sig_oid; /**< Signature algorithm, e.g. sha1RSA */ - - mbedtls_x509_buf issuer_raw; /**< The raw issuer data (DER). Used for quick comparison. */ - mbedtls_x509_buf subject_raw; /**< The raw subject data (DER). Used for quick comparison. */ - - mbedtls_x509_name issuer; /**< The parsed issuer data (named information object). */ - mbedtls_x509_name subject; /**< The parsed subject data (named information object). */ - - mbedtls_x509_time valid_from; /**< Start time of certificate validity. */ - mbedtls_x509_time valid_to; /**< End time of certificate validity. */ - - mbedtls_x509_buf pk_raw; - mbedtls_pk_context pk; /**< Container for the public key context. */ - - mbedtls_x509_buf issuer_id; /**< Optional X.509 v2/v3 issuer unique identifier. */ - mbedtls_x509_buf subject_id; /**< Optional X.509 v2/v3 subject unique identifier. */ - mbedtls_x509_buf v3_ext; /**< Optional X.509 v3 extensions. */ - mbedtls_x509_sequence subject_alt_names; /**< Optional list of raw entries of Subject Alternative Names extension. These can be later parsed by mbedtls_x509_parse_subject_alt_name. */ - mbedtls_x509_buf subject_key_id; /**< Optional X.509 v3 extension subject key identifier. */ - mbedtls_x509_authority authority_key_id; /**< Optional X.509 v3 extension authority key identifier. */ - - mbedtls_x509_sequence certificate_policies; /**< Optional list of certificate policies (Only anyPolicy is printed and enforced, however the rest of the policies are still listed). */ - - int MBEDTLS_PRIVATE(ext_types); /**< Bit string containing detected and parsed extensions */ - int MBEDTLS_PRIVATE(ca_istrue); /**< Optional Basic Constraint extension value: 1 if this certificate belongs to a CA, 0 otherwise. */ - int MBEDTLS_PRIVATE(max_pathlen); /**< Optional Basic Constraint extension value: The maximum path length to the root certificate. Path length is 1 higher than RFC 5280 'meaning', so 1+ */ - - unsigned int MBEDTLS_PRIVATE(key_usage); /**< Optional key usage extension value: See the values in x509.h */ - - mbedtls_x509_sequence ext_key_usage; /**< Optional list of extended key usage OIDs. */ - - unsigned char MBEDTLS_PRIVATE(ns_cert_type); /**< Optional Netscape certificate type extension value: See the values in x509.h */ - - mbedtls_x509_buf MBEDTLS_PRIVATE(sig); /**< Signature: hash of the tbs part signed with the private key. */ - mbedtls_md_type_t MBEDTLS_PRIVATE(sig_md); /**< Internal representation of the MD algorithm of the signature algorithm, e.g. MBEDTLS_MD_SHA256 */ - mbedtls_pk_type_t MBEDTLS_PRIVATE(sig_pk); /**< Internal representation of the Public Key algorithm of the signature algorithm, e.g. MBEDTLS_PK_RSA */ - void *MBEDTLS_PRIVATE(sig_opts); /**< Signature options to be passed to mbedtls_pk_verify_ext(), e.g. for RSASSA-PSS */ - - /** Next certificate in the linked list that constitutes the CA chain. - * \p NULL indicates the end of the list. - * Do not modify this field directly. */ - struct mbedtls_x509_crt *next; -} -mbedtls_x509_crt; - -/** - * Build flag from an algorithm/curve identifier (pk, md, ecp) - * Since 0 is always XXX_NONE, ignore it. - */ -#define MBEDTLS_X509_ID_FLAG(id) (1 << ((id) - 1)) - -/** - * Security profile for certificate verification. - * - * All lists are bitfields, built by ORing flags from MBEDTLS_X509_ID_FLAG(). - * - * The fields of this structure are part of the public API and can be - * manipulated directly by applications. Future versions of the library may - * add extra fields or reorder existing fields. - * - * You can create custom profiles by starting from a copy of - * an existing profile, such as mbedtls_x509_crt_profile_default or - * mbedtls_x509_ctr_profile_none and then tune it to your needs. - * - * For example to allow SHA-224 in addition to the default: - * - * mbedtls_x509_crt_profile my_profile = mbedtls_x509_crt_profile_default; - * my_profile.allowed_mds |= MBEDTLS_X509_ID_FLAG( MBEDTLS_MD_SHA224 ); - * - * Or to allow only RSA-3072+ with SHA-256: - * - * mbedtls_x509_crt_profile my_profile = mbedtls_x509_crt_profile_none; - * my_profile.allowed_mds = MBEDTLS_X509_ID_FLAG( MBEDTLS_MD_SHA256 ); - * my_profile.allowed_pks = MBEDTLS_X509_ID_FLAG( MBEDTLS_PK_RSA ); - * my_profile.rsa_min_bitlen = 3072; - */ -typedef struct mbedtls_x509_crt_profile { - uint32_t allowed_mds; /**< MDs for signatures */ - uint32_t allowed_pks; /**< PK algs for public keys; - * this applies to all certificates - * in the provided chain. */ - uint32_t allowed_curves; /**< Elliptic curves for ECDSA */ - uint32_t rsa_min_bitlen; /**< Minimum size for RSA keys */ -} -mbedtls_x509_crt_profile; - -#define MBEDTLS_X509_CRT_VERSION_1 0 -#define MBEDTLS_X509_CRT_VERSION_2 1 -#define MBEDTLS_X509_CRT_VERSION_3 2 - -#define MBEDTLS_X509_RFC5280_MAX_SERIAL_LEN 20 -#define MBEDTLS_X509_RFC5280_UTC_TIME_LEN 15 - -#if !defined(MBEDTLS_X509_MAX_FILE_PATH_LEN) -#define MBEDTLS_X509_MAX_FILE_PATH_LEN 512 -#endif - -/* This macro unfolds to the concatenation of macro invocations - * X509_CRT_ERROR_INFO( error code, - * error code as string, - * human readable description ) - * where X509_CRT_ERROR_INFO is defined by the user. - * See x509_crt.c for an example of how to use this. */ -#define MBEDTLS_X509_CRT_ERROR_INFO_LIST \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_EXPIRED, \ - "MBEDTLS_X509_BADCERT_EXPIRED", \ - "The certificate validity has expired") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_REVOKED, \ - "MBEDTLS_X509_BADCERT_REVOKED", \ - "The certificate has been revoked (is on a CRL)") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_CN_MISMATCH, \ - "MBEDTLS_X509_BADCERT_CN_MISMATCH", \ - "The certificate Common Name (CN) does not match with the expected CN") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_NOT_TRUSTED, \ - "MBEDTLS_X509_BADCERT_NOT_TRUSTED", \ - "The certificate is not correctly signed by the trusted CA") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_NOT_TRUSTED, \ - "MBEDTLS_X509_BADCRL_NOT_TRUSTED", \ - "The CRL is not correctly signed by the trusted CA") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_EXPIRED, \ - "MBEDTLS_X509_BADCRL_EXPIRED", \ - "The CRL is expired") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_MISSING, \ - "MBEDTLS_X509_BADCERT_MISSING", \ - "Certificate was missing") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_SKIP_VERIFY, \ - "MBEDTLS_X509_BADCERT_SKIP_VERIFY", \ - "Certificate verification was skipped") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_OTHER, \ - "MBEDTLS_X509_BADCERT_OTHER", \ - "Other reason (can be used by verify callback)") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_FUTURE, \ - "MBEDTLS_X509_BADCERT_FUTURE", \ - "The certificate validity starts in the future") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_FUTURE, \ - "MBEDTLS_X509_BADCRL_FUTURE", \ - "The CRL is from the future") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_KEY_USAGE, \ - "MBEDTLS_X509_BADCERT_KEY_USAGE", \ - "Usage does not match the keyUsage extension") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_EXT_KEY_USAGE, \ - "MBEDTLS_X509_BADCERT_EXT_KEY_USAGE", \ - "Usage does not match the extendedKeyUsage extension") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_NS_CERT_TYPE, \ - "MBEDTLS_X509_BADCERT_NS_CERT_TYPE", \ - "Usage does not match the nsCertType extension") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_BAD_MD, \ - "MBEDTLS_X509_BADCERT_BAD_MD", \ - "The certificate is signed with an unacceptable hash.") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_BAD_PK, \ - "MBEDTLS_X509_BADCERT_BAD_PK", \ - "The certificate is signed with an unacceptable PK alg (eg RSA vs ECDSA).") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCERT_BAD_KEY, \ - "MBEDTLS_X509_BADCERT_BAD_KEY", \ - "The certificate is signed with an unacceptable key (eg bad curve, RSA too short).") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_BAD_MD, \ - "MBEDTLS_X509_BADCRL_BAD_MD", \ - "The CRL is signed with an unacceptable hash.") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_BAD_PK, \ - "MBEDTLS_X509_BADCRL_BAD_PK", \ - "The CRL is signed with an unacceptable PK alg (eg RSA vs ECDSA).") \ - X509_CRT_ERROR_INFO(MBEDTLS_X509_BADCRL_BAD_KEY, \ - "MBEDTLS_X509_BADCRL_BAD_KEY", \ - "The CRL is signed with an unacceptable key (eg bad curve, RSA too short).") - -/** - * Container for writing a certificate (CRT) - */ -typedef struct mbedtls_x509write_cert { - int MBEDTLS_PRIVATE(version); - unsigned char MBEDTLS_PRIVATE(serial)[MBEDTLS_X509_RFC5280_MAX_SERIAL_LEN]; - size_t MBEDTLS_PRIVATE(serial_len); - mbedtls_pk_context *MBEDTLS_PRIVATE(subject_key); - mbedtls_pk_context *MBEDTLS_PRIVATE(issuer_key); - mbedtls_asn1_named_data *MBEDTLS_PRIVATE(subject); - mbedtls_asn1_named_data *MBEDTLS_PRIVATE(issuer); - mbedtls_md_type_t MBEDTLS_PRIVATE(md_alg); - char MBEDTLS_PRIVATE(not_before)[MBEDTLS_X509_RFC5280_UTC_TIME_LEN + 1]; - char MBEDTLS_PRIVATE(not_after)[MBEDTLS_X509_RFC5280_UTC_TIME_LEN + 1]; - mbedtls_asn1_named_data *MBEDTLS_PRIVATE(extensions); -} -mbedtls_x509write_cert; - -/** - * \brief Set Subject Alternative Name - * - * \param ctx Certificate context to use - * \param san_list List of SAN values - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - * - * \note "dnsName", "uniformResourceIdentifier", "IP address", - * "otherName", and "DirectoryName", as defined in RFC 5280, - * are supported. - */ -int mbedtls_x509write_crt_set_subject_alternative_name(mbedtls_x509write_cert *ctx, - const mbedtls_x509_san_list *san_list); - -/** - * Item in a verification chain: cert and flags for it - */ -typedef struct { - mbedtls_x509_crt *MBEDTLS_PRIVATE(crt); - uint32_t MBEDTLS_PRIVATE(flags); -} mbedtls_x509_crt_verify_chain_item; - -/** - * Max size of verification chain: end-entity + intermediates + trusted root - */ -#define MBEDTLS_X509_MAX_VERIFY_CHAIN_SIZE (MBEDTLS_X509_MAX_INTERMEDIATE_CA + 2) - -/** - * Verification chain as built by \c mbedtls_crt_verify_chain() - */ -typedef struct { - mbedtls_x509_crt_verify_chain_item MBEDTLS_PRIVATE(items)[MBEDTLS_X509_MAX_VERIFY_CHAIN_SIZE]; - unsigned MBEDTLS_PRIVATE(len); - -#if defined(MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK) - /* This stores the list of potential trusted signers obtained from - * the CA callback used for the CRT verification, if configured. - * We must track it somewhere because the callback passes its - * ownership to the caller. */ - mbedtls_x509_crt *MBEDTLS_PRIVATE(trust_ca_cb_result); -#endif /* MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK */ -} mbedtls_x509_crt_verify_chain; - -#if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) - -/** - * \brief Context for resuming X.509 verify operations - */ -typedef struct { - /* for check_signature() */ - mbedtls_pk_restart_ctx MBEDTLS_PRIVATE(pk); - - /* for find_parent_in() */ - mbedtls_x509_crt *MBEDTLS_PRIVATE(parent); /* non-null iff parent_in in progress */ - mbedtls_x509_crt *MBEDTLS_PRIVATE(fallback_parent); - int MBEDTLS_PRIVATE(fallback_signature_is_good); - - /* for find_parent() */ - int MBEDTLS_PRIVATE(parent_is_trusted); /* -1 if find_parent is not in progress */ - - /* for verify_chain() */ - enum { - x509_crt_rs_none, - x509_crt_rs_find_parent, - } MBEDTLS_PRIVATE(in_progress); /* none if no operation is in progress */ - int MBEDTLS_PRIVATE(self_cnt); - mbedtls_x509_crt_verify_chain MBEDTLS_PRIVATE(ver_chain); - -} mbedtls_x509_crt_restart_ctx; - -#else /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ - -/* Now we can declare functions that take a pointer to that */ -typedef void mbedtls_x509_crt_restart_ctx; - -#endif /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ - -#if defined(MBEDTLS_X509_CRT_PARSE_C) -/** - * Default security profile. Should provide a good balance between security - * and compatibility with current deployments. - * - * This profile permits: - * - SHA2 hashes with at least 256 bits: SHA-256, SHA-384, SHA-512. - * - Elliptic curves with 255 bits and above except secp256k1. - * - RSA with 2048 bits and above. - * - * New minor versions of Mbed TLS may extend this profile, for example if - * new algorithms are added to the library. New minor versions of Mbed TLS will - * not reduce this profile unless serious security concerns require it. - */ -extern const mbedtls_x509_crt_profile mbedtls_x509_crt_profile_default; - -/** - * Expected next default profile. Recommended for new deployments. - * Currently targets a 128-bit security level, except for allowing RSA-2048. - * This profile may change at any time. - */ -extern const mbedtls_x509_crt_profile mbedtls_x509_crt_profile_next; - -/** - * NSA Suite B profile. - */ -extern const mbedtls_x509_crt_profile mbedtls_x509_crt_profile_suiteb; - -/** - * Empty profile that allows nothing. Useful as a basis for constructing - * custom profiles. - */ -extern const mbedtls_x509_crt_profile mbedtls_x509_crt_profile_none; - -/** - * \brief Parse a single DER formatted certificate and add it - * to the end of the provided chained list. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain The pointer to the start of the CRT chain to attach to. - * When parsing the first CRT in a chain, this should point - * to an instance of ::mbedtls_x509_crt initialized through - * mbedtls_x509_crt_init(). - * \param buf The buffer holding the DER encoded certificate. - * \param buflen The size in Bytes of \p buf. - * - * \note This function makes an internal copy of the CRT buffer - * \p buf. In particular, \p buf may be destroyed or reused - * after this call returns. To avoid duplicating the CRT - * buffer (at the cost of stricter lifetime constraints), - * use mbedtls_x509_crt_parse_der_nocopy() instead. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_x509_crt_parse_der(mbedtls_x509_crt *chain, - const unsigned char *buf, - size_t buflen); - -/** - * \brief The type of certificate extension callbacks. - * - * Callbacks of this type are passed to and used by the - * mbedtls_x509_crt_parse_der_with_ext_cb() routine when - * it encounters either an unsupported extension or a - * "certificate policies" extension containing any - * unsupported certificate policies. - * Future versions of the library may invoke the callback - * in other cases, if and when the need arises. - * - * \param p_ctx An opaque context passed to the callback. - * \param crt The certificate being parsed. - * \param oid The OID of the extension. - * \param critical Whether the extension is critical. - * \param p Pointer to the start of the extension value - * (the content of the OCTET STRING). - * \param end End of extension value. - * - * \note The callback must fail and return a negative error code - * if it can not parse or does not support the extension. - * When the callback fails to parse a critical extension - * mbedtls_x509_crt_parse_der_with_ext_cb() also fails. - * When the callback fails to parse a non critical extension - * mbedtls_x509_crt_parse_der_with_ext_cb() simply skips - * the extension and continues parsing. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -typedef int (*mbedtls_x509_crt_ext_cb_t)(void *p_ctx, - mbedtls_x509_crt const *crt, - mbedtls_x509_buf const *oid, - int critical, - const unsigned char *p, - const unsigned char *end); - -/** - * \brief Parse a single DER formatted certificate and add it - * to the end of the provided chained list. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain The pointer to the start of the CRT chain to attach to. - * When parsing the first CRT in a chain, this should point - * to an instance of ::mbedtls_x509_crt initialized through - * mbedtls_x509_crt_init(). - * \param buf The buffer holding the DER encoded certificate. - * \param buflen The size in Bytes of \p buf. - * \param make_copy When not zero this function makes an internal copy of the - * CRT buffer \p buf. In particular, \p buf may be destroyed - * or reused after this call returns. - * When zero this function avoids duplicating the CRT buffer - * by taking temporary ownership thereof until the CRT - * is destroyed (like mbedtls_x509_crt_parse_der_nocopy()) - * \param cb A callback invoked for every unsupported certificate - * extension. - * \param p_ctx An opaque context passed to the callback. - * - * \note This call is functionally equivalent to - * mbedtls_x509_crt_parse_der(), and/or - * mbedtls_x509_crt_parse_der_nocopy() - * but it calls the callback with every unsupported - * certificate extension and additionally the - * "certificate policies" extension if it contains any - * unsupported certificate policies. - * The callback must return a negative error code if it - * does not know how to handle such an extension. - * When the callback fails to parse a critical extension - * mbedtls_x509_crt_parse_der_with_ext_cb() also fails. - * When the callback fails to parse a non critical extension - * mbedtls_x509_crt_parse_der_with_ext_cb() simply skips - * the extension and continues parsing. - * Future versions of the library may invoke the callback - * in other cases, if and when the need arises. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_x509_crt_parse_der_with_ext_cb(mbedtls_x509_crt *chain, - const unsigned char *buf, - size_t buflen, - int make_copy, - mbedtls_x509_crt_ext_cb_t cb, - void *p_ctx); - -/** - * \brief Parse a single DER formatted certificate and add it - * to the end of the provided chained list. This is a - * variant of mbedtls_x509_crt_parse_der() which takes - * temporary ownership of the CRT buffer until the CRT - * is destroyed. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain The pointer to the start of the CRT chain to attach to. - * When parsing the first CRT in a chain, this should point - * to an instance of ::mbedtls_x509_crt initialized through - * mbedtls_x509_crt_init(). - * \param buf The address of the readable buffer holding the DER encoded - * certificate to use. On success, this buffer must be - * retained and not be changed for the lifetime of the - * CRT chain \p chain, that is, until \p chain is destroyed - * through a call to mbedtls_x509_crt_free(). - * \param buflen The size in Bytes of \p buf. - * - * \note This call is functionally equivalent to - * mbedtls_x509_crt_parse_der(), but it avoids creating a - * copy of the input buffer at the cost of stronger lifetime - * constraints. This is useful in constrained environments - * where duplication of the CRT cannot be tolerated. - * - * \return \c 0 if successful. - * \return A negative error code on failure. - */ -int mbedtls_x509_crt_parse_der_nocopy(mbedtls_x509_crt *chain, - const unsigned char *buf, - size_t buflen); - -/** - * \brief Parse one DER-encoded or one or more concatenated PEM-encoded - * certificates and add them to the chained list. - * - * For CRTs in PEM encoding, the function parses permissively: - * if at least one certificate can be parsed, the function - * returns the number of certificates for which parsing failed - * (hence \c 0 if all certificates were parsed successfully). - * If no certificate could be parsed, the function returns - * the first (negative) error encountered during parsing. - * - * PEM encoded certificates may be interleaved by other data - * such as human readable descriptions of their content, as - * long as the certificates are enclosed in the PEM specific - * '-----{BEGIN/END} CERTIFICATE-----' delimiters. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain The chain to which to add the parsed certificates. - * \param buf The buffer holding the certificate data in PEM or DER format. - * For certificates in PEM encoding, this may be a concatenation - * of multiple certificates; for DER encoding, the buffer must - * comprise exactly one certificate. - * \param buflen The size of \p buf, including the terminating \c NULL byte - * in case of PEM encoded data. - * - * \return \c 0 if all certificates were parsed successfully. - * \return The (positive) number of certificates that couldn't - * be parsed if parsing was partly successful (see above). - * \return A negative X509 or PEM error code otherwise. - * - */ -int mbedtls_x509_crt_parse(mbedtls_x509_crt *chain, const unsigned char *buf, size_t buflen); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief Load one or more certificates and add them - * to the chained list. Parses permissively. If some - * certificates can be parsed, the result is the number - * of failed certificates it encountered. If none complete - * correctly, the first error is returned. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param chain points to the start of the chain - * \param path filename to read the certificates from - * - * \return 0 if all certificates parsed successfully, a positive number - * if partly successful or a specific X509 or PEM error code - */ -int mbedtls_x509_crt_parse_file(mbedtls_x509_crt *chain, const char *path); - -/** - * \brief Load one or more certificate files from a path and add them - * to the chained list. Parses permissively. If some - * certificates can be parsed, the result is the number - * of failed certificates it encountered. If none complete - * correctly, the first error is returned. - * - * \param chain points to the start of the chain - * \param path directory / folder to read the certificate files from - * - * \return 0 if all certificates parsed successfully, a positive number - * if partly successful or a specific X509 or PEM error code - */ -int mbedtls_x509_crt_parse_path(mbedtls_x509_crt *chain, const char *path); - -#endif /* MBEDTLS_FS_IO */ - -#if !defined(MBEDTLS_X509_REMOVE_INFO) -/** - * \brief Returns an informational string about the - * certificate. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param prefix A line prefix - * \param crt The X509 certificate to represent - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_crt_info(char *buf, size_t size, const char *prefix, - const mbedtls_x509_crt *crt); - -/** - * \brief Returns an informational string about the - * verification status of a certificate. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param prefix A line prefix - * \param flags Verification flags created by mbedtls_x509_crt_verify() - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_crt_verify_info(char *buf, size_t size, const char *prefix, - uint32_t flags); -#endif /* !MBEDTLS_X509_REMOVE_INFO */ - -/** - * \brief Verify a chain of certificates. - * - * The verify callback is a user-supplied callback that - * can clear / modify / add flags for a certificate. If set, - * the verification callback is called for each - * certificate in the chain (from the trust-ca down to the - * presented crt). The parameters for the callback are: - * (void *parameter, mbedtls_x509_crt *crt, int certificate_depth, - * int *flags). With the flags representing current flags for - * that specific certificate and the certificate depth from - * the bottom (Peer cert depth = 0). - * - * All flags left after returning from the callback - * are also returned to the application. The function should - * return 0 for anything (including invalid certificates) - * other than fatal error, as a non-zero return code - * immediately aborts the verification process. For fatal - * errors, a specific error code should be used (different - * from MBEDTLS_ERR_X509_CERT_VERIFY_FAILED which should not - * be returned at this point), or MBEDTLS_ERR_X509_FATAL_ERROR - * can be used if no better code is available. - * - * \note In case verification failed, the results can be displayed - * using \c mbedtls_x509_crt_verify_info() - * - * \note Same as \c mbedtls_x509_crt_verify_with_profile() with the - * default security profile. - * - * \note It is your responsibility to provide up-to-date CRLs for - * all trusted CAs. If no CRL is provided for the CA that was - * used to sign the certificate, CRL verification is skipped - * silently, that is *without* setting any flag. - * - * \note The \c trust_ca list can contain two types of certificates: - * (1) those of trusted root CAs, so that certificates - * chaining up to those CAs will be trusted, and (2) - * self-signed end-entity certificates to be trusted (for - * specific peers you know) - in that case, the self-signed - * certificate doesn't need to have the CA bit set. - * - * \param crt The certificate chain to be verified. - * \param trust_ca The list of trusted CAs. - * \param ca_crl The list of CRLs for trusted CAs. - * \param cn The expected Common Name. This will be checked to be - * present in the certificate's subjectAltNames extension or, - * if this extension is absent, as a CN component in its - * Subject name. DNS names and IP addresses are fully - * supported, while the URI subtype is partially supported: - * only exact matching, without any normalization procedures - * described in 7.4 of RFC5280, will result in a positive - * URI verification. - * This may be \c NULL if the CN need not be verified. - * \param flags The address at which to store the result of the verification. - * If the verification couldn't be completed, the flag value is - * set to (uint32_t) -1. - * \param f_vrfy The verification callback to use. See the documentation - * of mbedtls_x509_crt_verify() for more information. - * \param p_vrfy The context to be passed to \p f_vrfy. - * - * \return \c 0 if the chain is valid with respect to the - * passed CN, CAs, CRLs and security profile. - * \return #MBEDTLS_ERR_X509_CERT_VERIFY_FAILED in case the - * certificate chain verification failed. In this case, - * \c *flags will have one or more - * \c MBEDTLS_X509_BADCERT_XXX or \c MBEDTLS_X509_BADCRL_XXX - * flags set. - * \return Another negative error code in case of a fatal error - * encountered during the verification process. - */ -int mbedtls_x509_crt_verify(mbedtls_x509_crt *crt, - mbedtls_x509_crt *trust_ca, - mbedtls_x509_crl *ca_crl, - const char *cn, uint32_t *flags, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy); - -/** - * \brief Verify a chain of certificates with respect to - * a configurable security profile. - * - * \note Same as \c mbedtls_x509_crt_verify(), but with explicit - * security profile. - * - * \note The restrictions on keys (RSA minimum size, allowed curves - * for ECDSA) apply to all certificates: trusted root, - * intermediate CAs if any, and end entity certificate. - * - * \param crt The certificate chain to be verified. - * \param trust_ca The list of trusted CAs. - * \param ca_crl The list of CRLs for trusted CAs. - * \param profile The security profile to use for the verification. - * \param cn The expected Common Name. This may be \c NULL if the - * CN need not be verified. - * \param flags The address at which to store the result of the verification. - * If the verification couldn't be completed, the flag value is - * set to (uint32_t) -1. - * \param f_vrfy The verification callback to use. See the documentation - * of mbedtls_x509_crt_verify() for more information. - * \param p_vrfy The context to be passed to \p f_vrfy. - * - * \return \c 0 if the chain is valid with respect to the - * passed CN, CAs, CRLs and security profile. - * \return #MBEDTLS_ERR_X509_CERT_VERIFY_FAILED in case the - * certificate chain verification failed. In this case, - * \c *flags will have one or more - * \c MBEDTLS_X509_BADCERT_XXX or \c MBEDTLS_X509_BADCRL_XXX - * flags set. - * \return Another negative error code in case of a fatal error - * encountered during the verification process. - */ -int mbedtls_x509_crt_verify_with_profile(mbedtls_x509_crt *crt, - mbedtls_x509_crt *trust_ca, - mbedtls_x509_crl *ca_crl, - const mbedtls_x509_crt_profile *profile, - const char *cn, uint32_t *flags, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy); - -/** - * \brief Restartable version of \c mbedtls_crt_verify_with_profile() - * - * \note Performs the same job as \c mbedtls_crt_verify_with_profile() - * but can return early and restart according to the limit - * set with \c mbedtls_ecp_set_max_ops() to reduce blocking. - * - * \param crt The certificate chain to be verified. - * \param trust_ca The list of trusted CAs. - * \param ca_crl The list of CRLs for trusted CAs. - * \param profile The security profile to use for the verification. - * \param cn The expected Common Name. This may be \c NULL if the - * CN need not be verified. - * \param flags The address at which to store the result of the verification. - * If the verification couldn't be completed, the flag value is - * set to (uint32_t) -1. - * \param f_vrfy The verification callback to use. See the documentation - * of mbedtls_x509_crt_verify() for more information. - * \param p_vrfy The context to be passed to \p f_vrfy. - * \param rs_ctx The restart context to use. This may be set to \c NULL - * to disable restartable ECC. - * - * \return See \c mbedtls_crt_verify_with_profile(), or - * \return #MBEDTLS_ERR_ECP_IN_PROGRESS if maximum number of - * operations was reached: see \c mbedtls_ecp_set_max_ops(). - */ -int mbedtls_x509_crt_verify_restartable(mbedtls_x509_crt *crt, - mbedtls_x509_crt *trust_ca, - mbedtls_x509_crl *ca_crl, - const mbedtls_x509_crt_profile *profile, - const char *cn, uint32_t *flags, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy, - mbedtls_x509_crt_restart_ctx *rs_ctx); - -/** - * \brief The type of trusted certificate callbacks. - * - * Callbacks of this type are passed to and used by the CRT - * verification routine mbedtls_x509_crt_verify_with_ca_cb() - * when looking for trusted signers of a given certificate. - * - * On success, the callback returns a list of trusted - * certificates to be considered as potential signers - * for the input certificate. - * - * \param p_ctx An opaque context passed to the callback. - * \param child The certificate for which to search a potential signer. - * This will point to a readable certificate. - * \param candidate_cas The address at which to store the address of the first - * entry in the generated linked list of candidate signers. - * This will not be \c NULL. - * - * \note The callback must only return a non-zero value on a - * fatal error. If, in contrast, the search for a potential - * signer completes without a single candidate, the - * callback must return \c 0 and set \c *candidate_cas - * to \c NULL. - * - * \return \c 0 on success. In this case, \c *candidate_cas points - * to a heap-allocated linked list of instances of - * ::mbedtls_x509_crt, and ownership of this list is passed - * to the caller. - * \return A negative error code on failure. - */ -typedef int (*mbedtls_x509_crt_ca_cb_t)(void *p_ctx, - mbedtls_x509_crt const *child, - mbedtls_x509_crt **candidate_cas); - -#if defined(MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK) -/** - * \brief Version of \c mbedtls_x509_crt_verify_with_profile() which - * uses a callback to acquire the list of trusted CA - * certificates. - * - * \param crt The certificate chain to be verified. - * \param f_ca_cb The callback to be used to query for potential signers - * of a given child certificate. See the documentation of - * ::mbedtls_x509_crt_ca_cb_t for more information. - * \param p_ca_cb The opaque context to be passed to \p f_ca_cb. - * \param profile The security profile for the verification. - * \param cn The expected Common Name. This may be \c NULL if the - * CN need not be verified. - * \param flags The address at which to store the result of the verification. - * If the verification couldn't be completed, the flag value is - * set to (uint32_t) -1. - * \param f_vrfy The verification callback to use. See the documentation - * of mbedtls_x509_crt_verify() for more information. - * \param p_vrfy The context to be passed to \p f_vrfy. - * - * \return See \c mbedtls_crt_verify_with_profile(). - */ -int mbedtls_x509_crt_verify_with_ca_cb(mbedtls_x509_crt *crt, - mbedtls_x509_crt_ca_cb_t f_ca_cb, - void *p_ca_cb, - const mbedtls_x509_crt_profile *profile, - const char *cn, uint32_t *flags, - int (*f_vrfy)(void *, mbedtls_x509_crt *, int, uint32_t *), - void *p_vrfy); - -#endif /* MBEDTLS_X509_TRUSTED_CERTIFICATE_CALLBACK */ - -/** - * \brief Check usage of certificate against keyUsage extension. - * - * \param crt Leaf certificate used. - * \param usage Intended usage(s) (eg MBEDTLS_X509_KU_KEY_ENCIPHERMENT - * before using the certificate to perform an RSA key - * exchange). - * - * \note Except for decipherOnly and encipherOnly, a bit set in the - * usage argument means this bit MUST be set in the - * certificate. For decipherOnly and encipherOnly, it means - * that bit MAY be set. - * - * \return 0 is these uses of the certificate are allowed, - * MBEDTLS_ERR_X509_BAD_INPUT_DATA if the keyUsage extension - * is present but does not match the usage argument. - * - * \note You should only call this function on leaf certificates, on - * (intermediate) CAs the keyUsage extension is automatically - * checked by \c mbedtls_x509_crt_verify(). - */ -int mbedtls_x509_crt_check_key_usage(const mbedtls_x509_crt *crt, - unsigned int usage); - -/** - * \brief Check usage of certificate against extendedKeyUsage. - * - * \param crt Leaf certificate used. - * \param usage_oid Intended usage (eg MBEDTLS_OID_SERVER_AUTH or - * MBEDTLS_OID_CLIENT_AUTH). - * \param usage_len Length of usage_oid (eg given by MBEDTLS_OID_SIZE()). - * - * \return 0 if this use of the certificate is allowed, - * MBEDTLS_ERR_X509_BAD_INPUT_DATA if not. - * - * \note Usually only makes sense on leaf certificates. - */ -int mbedtls_x509_crt_check_extended_key_usage(const mbedtls_x509_crt *crt, - const char *usage_oid, - size_t usage_len); - -#if defined(MBEDTLS_X509_CRL_PARSE_C) -/** - * \brief Verify the certificate revocation status - * - * \param crt a certificate to be verified - * \param crl the CRL to verify against - * - * \return 1 if the certificate is revoked, 0 otherwise - * - */ -int mbedtls_x509_crt_is_revoked(const mbedtls_x509_crt *crt, const mbedtls_x509_crl *crl); -#endif /* MBEDTLS_X509_CRL_PARSE_C */ - -/** - * \brief Initialize a certificate (chain) - * - * \param crt Certificate chain to initialize - */ -void mbedtls_x509_crt_init(mbedtls_x509_crt *crt); - -/** - * \brief Unallocate all certificate data - * - * \param crt Certificate chain to free - */ -void mbedtls_x509_crt_free(mbedtls_x509_crt *crt); - -#if defined(MBEDTLS_ECDSA_C) && defined(MBEDTLS_ECP_RESTARTABLE) -/** - * \brief Initialize a restart context - */ -void mbedtls_x509_crt_restart_init(mbedtls_x509_crt_restart_ctx *ctx); - -/** - * \brief Free the components of a restart context - */ -void mbedtls_x509_crt_restart_free(mbedtls_x509_crt_restart_ctx *ctx); -#endif /* MBEDTLS_ECDSA_C && MBEDTLS_ECP_RESTARTABLE */ -#endif /* MBEDTLS_X509_CRT_PARSE_C */ - -/** - * \brief Query certificate for given extension type - * - * \param[in] ctx Certificate context to be queried, must not be \c NULL - * \param ext_type Extension type being queried for, must be a valid - * extension type. Must be one of the MBEDTLS_X509_EXT_XXX - * values - * - * \return 0 if the given extension type is not present, - * non-zero otherwise - */ -static inline int mbedtls_x509_crt_has_ext_type(const mbedtls_x509_crt *ctx, - int ext_type) -{ - return ctx->MBEDTLS_PRIVATE(ext_types) & ext_type; -} - -/** - * \brief Access the ca_istrue field - * - * \param[in] crt Certificate to be queried, must not be \c NULL - * - * \return \c 1 if this a CA certificate \c 0 otherwise. - * \return MBEDTLS_ERR_X509_INVALID_EXTENSIONS if the certificate does not contain - * the Optional Basic Constraint extension. - * - */ -int mbedtls_x509_crt_get_ca_istrue(const mbedtls_x509_crt *crt); - -/** \} name Structures and functions for parsing and writing X.509 certificates */ - -#if defined(MBEDTLS_X509_CRT_WRITE_C) -/** - * \brief Initialize a CRT writing context - * - * \param ctx CRT context to initialize - */ -void mbedtls_x509write_crt_init(mbedtls_x509write_cert *ctx); - -/** - * \brief Set the version for a Certificate - * Default: MBEDTLS_X509_CRT_VERSION_3 - * - * \param ctx CRT context to use - * \param version version to set (MBEDTLS_X509_CRT_VERSION_1, MBEDTLS_X509_CRT_VERSION_2 or - * MBEDTLS_X509_CRT_VERSION_3) - */ -void mbedtls_x509write_crt_set_version(mbedtls_x509write_cert *ctx, int version); - -#if defined(MBEDTLS_BIGNUM_C) && !defined(MBEDTLS_DEPRECATED_REMOVED) -/** - * \brief Set the serial number for a Certificate. - * - * \deprecated This function is deprecated and will be removed in a - * future version of the library. Please use - * mbedtls_x509write_crt_set_serial_raw() instead. - * - * \note Even though the MBEDTLS_BIGNUM_C guard looks redundant since - * X509 depends on PK and PK depends on BIGNUM, this emphasizes - * a direct dependency between X509 and BIGNUM which is going - * to be deprecated in the future. - * - * \param ctx CRT context to use - * \param serial serial number to set - * - * \return 0 if successful - */ -int MBEDTLS_DEPRECATED mbedtls_x509write_crt_set_serial( - mbedtls_x509write_cert *ctx, const mbedtls_mpi *serial); -#endif // MBEDTLS_BIGNUM_C && !MBEDTLS_DEPRECATED_REMOVED - -/** - * \brief Set the serial number for a Certificate. - * - * \param ctx CRT context to use - * \param serial A raw array of bytes containing the serial number in big - * endian format - * \param serial_len Length of valid bytes (expressed in bytes) in \p serial - * input buffer - * - * \return 0 if successful, or - * MBEDTLS_ERR_X509_BAD_INPUT_DATA if the provided input buffer - * is too big (longer than MBEDTLS_X509_RFC5280_MAX_SERIAL_LEN) - */ -int mbedtls_x509write_crt_set_serial_raw(mbedtls_x509write_cert *ctx, - unsigned char *serial, size_t serial_len); - -/** - * \brief Set the validity period for a Certificate - * Timestamps should be in string format for UTC timezone - * i.e. "YYYYMMDDhhmmss" - * e.g. "20131231235959" for December 31st 2013 - * at 23:59:59 - * - * \param ctx CRT context to use - * \param not_before not_before timestamp - * \param not_after not_after timestamp - * - * \return 0 if timestamp was parsed successfully, or - * a specific error code - */ -int mbedtls_x509write_crt_set_validity(mbedtls_x509write_cert *ctx, const char *not_before, - const char *not_after); - -/** - * \brief Set the issuer name for a Certificate - * Issuer names should contain a comma-separated list - * of OID types and values: - * e.g. "C=UK,O=ARM,CN=Mbed TLS CA" - * - * \param ctx CRT context to use - * \param issuer_name issuer name to set - * - * \return 0 if issuer name was parsed successfully, or - * a specific error code - */ -int mbedtls_x509write_crt_set_issuer_name(mbedtls_x509write_cert *ctx, - const char *issuer_name); - -/** - * \brief Set the subject name for a Certificate - * Subject names should contain a comma-separated list - * of OID types and values: - * e.g. "C=UK,O=ARM,CN=Mbed TLS Server 1" - * - * \param ctx CRT context to use - * \param subject_name subject name to set - * - * \return 0 if subject name was parsed successfully, or - * a specific error code - */ -int mbedtls_x509write_crt_set_subject_name(mbedtls_x509write_cert *ctx, - const char *subject_name); - -/** - * \brief Set the subject public key for the certificate - * - * \param ctx CRT context to use - * \param key public key to include - */ -void mbedtls_x509write_crt_set_subject_key(mbedtls_x509write_cert *ctx, mbedtls_pk_context *key); - -/** - * \brief Set the issuer key used for signing the certificate - * - * \param ctx CRT context to use - * \param key private key to sign with - */ -void mbedtls_x509write_crt_set_issuer_key(mbedtls_x509write_cert *ctx, mbedtls_pk_context *key); - -/** - * \brief Set the MD algorithm to use for the signature - * (e.g. MBEDTLS_MD_SHA1) - * - * \param ctx CRT context to use - * \param md_alg MD algorithm to use - */ -void mbedtls_x509write_crt_set_md_alg(mbedtls_x509write_cert *ctx, mbedtls_md_type_t md_alg); - -/** - * \brief Generic function to add to or replace an extension in the - * CRT - * - * \param ctx CRT context to use - * \param oid OID of the extension - * \param oid_len length of the OID - * \param critical if the extension is critical (per the RFC's definition) - * \param val value of the extension OCTET STRING - * \param val_len length of the value data - * - * \return 0 if successful, or a MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_extension(mbedtls_x509write_cert *ctx, - const char *oid, size_t oid_len, - int critical, - const unsigned char *val, size_t val_len); - -/** - * \brief Set the basicConstraints extension for a CRT - * - * \param ctx CRT context to use - * \param is_ca is this a CA certificate - * \param max_pathlen maximum length of certificate chains below this - * certificate (only for CA certificates, -1 is - * unlimited) - * - * \return 0 if successful, or a MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_basic_constraints(mbedtls_x509write_cert *ctx, - int is_ca, int max_pathlen); - -#if defined(MBEDTLS_MD_CAN_SHA1) -/** - * \brief Set the subjectKeyIdentifier extension for a CRT - * Requires that mbedtls_x509write_crt_set_subject_key() has been - * called before - * - * \param ctx CRT context to use - * - * \return 0 if successful, or a MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_subject_key_identifier(mbedtls_x509write_cert *ctx); - -/** - * \brief Set the authorityKeyIdentifier extension for a CRT - * Requires that mbedtls_x509write_crt_set_issuer_key() has been - * called before - * - * \param ctx CRT context to use - * - * \return 0 if successful, or a MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_authority_key_identifier(mbedtls_x509write_cert *ctx); -#endif /* MBEDTLS_MD_CAN_SHA1 */ - -/** - * \brief Set the Key Usage Extension flags - * (e.g. MBEDTLS_X509_KU_DIGITAL_SIGNATURE | MBEDTLS_X509_KU_KEY_CERT_SIGN) - * - * \param ctx CRT context to use - * \param key_usage key usage flags to set - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_key_usage(mbedtls_x509write_cert *ctx, - unsigned int key_usage); - -/** - * \brief Set the Extended Key Usage Extension - * (e.g. MBEDTLS_OID_SERVER_AUTH) - * - * \param ctx CRT context to use - * \param exts extended key usage extensions to set, a sequence of - * MBEDTLS_ASN1_OID objects - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_ext_key_usage(mbedtls_x509write_cert *ctx, - const mbedtls_asn1_sequence *exts); - -/** - * \brief Set the Netscape Cert Type flags - * (e.g. MBEDTLS_X509_NS_CERT_TYPE_SSL_CLIENT | MBEDTLS_X509_NS_CERT_TYPE_EMAIL) - * - * \param ctx CRT context to use - * \param ns_cert_type Netscape Cert Type flags to set - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_crt_set_ns_cert_type(mbedtls_x509write_cert *ctx, - unsigned char ns_cert_type); - -/** - * \brief Free the contents of a CRT write context - * - * \param ctx CRT context to free - */ -void mbedtls_x509write_crt_free(mbedtls_x509write_cert *ctx); - -/** - * \brief Write a built up certificate to a X509 DER structure - * Note: data is written at the end of the buffer! Use the - * return value to determine where you should start - * using the buffer - * - * \param ctx certificate to write away - * \param buf buffer to write to - * \param size size of the buffer - * \param f_rng RNG function. This must not be \c NULL. - * \param p_rng RNG parameter - * - * \return length of data written if successful, or a specific - * error code - * - * \note \p f_rng is used for the signature operation. - */ -int mbedtls_x509write_crt_der(mbedtls_x509write_cert *ctx, unsigned char *buf, size_t size, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -#if defined(MBEDTLS_PEM_WRITE_C) -/** - * \brief Write a built up certificate to a X509 PEM string - * - * \param ctx certificate to write away - * \param buf buffer to write to - * \param size size of the buffer - * \param f_rng RNG function. This must not be \c NULL. - * \param p_rng RNG parameter - * - * \return 0 if successful, or a specific error code - * - * \note \p f_rng is used for the signature operation. - */ -int mbedtls_x509write_crt_pem(mbedtls_x509write_cert *ctx, unsigned char *buf, size_t size, - mbedtls_f_rng_t *f_rng, - void *p_rng); -#endif /* MBEDTLS_PEM_WRITE_C */ -#endif /* MBEDTLS_X509_CRT_WRITE_C */ - -/** \} addtogroup x509_module */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_x509_crt.h */ diff --git a/android/mbedtls/arm64-v8a/include/mbedtls/x509_csr.h b/android/mbedtls/arm64-v8a/include/mbedtls/x509_csr.h deleted file mode 100644 index c4bd7f122..000000000 --- a/android/mbedtls/arm64-v8a/include/mbedtls/x509_csr.h +++ /dev/null @@ -1,382 +0,0 @@ -/** - * \file x509_csr.h - * - * \brief X.509 certificate signing request parsing and writing - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef MBEDTLS_X509_CSR_H -#define MBEDTLS_X509_CSR_H -#include "mbedtls/private_access.h" - -#include "mbedtls/build_info.h" - -#include "mbedtls/x509.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * \addtogroup x509_module - * \{ */ - -/** - * \name Structures and functions for X.509 Certificate Signing Requests (CSR) - * \{ - */ - -/** - * Certificate Signing Request (CSR) structure. - * - * Some fields of this structure are publicly readable. Do not modify - * them except via Mbed TLS library functions: the effect of modifying - * those fields or the data that those fields point to is unspecified. - */ -typedef struct mbedtls_x509_csr { - mbedtls_x509_buf raw; /**< The raw CSR data (DER). */ - mbedtls_x509_buf cri; /**< The raw CertificateRequestInfo body (DER). */ - - int version; /**< CSR version (1=v1). */ - - mbedtls_x509_buf subject_raw; /**< The raw subject data (DER). */ - mbedtls_x509_name subject; /**< The parsed subject data (named information object). */ - - mbedtls_pk_context pk; /**< Container for the public key context. */ - - unsigned int key_usage; /**< Optional key usage extension value: See the values in x509.h */ - unsigned char ns_cert_type; /**< Optional Netscape certificate type extension value: See the values in x509.h */ - mbedtls_x509_sequence subject_alt_names; /**< Optional list of raw entries of Subject Alternative Names extension. These can be later parsed by mbedtls_x509_parse_subject_alt_name. */ - - int MBEDTLS_PRIVATE(ext_types); /**< Bit string containing detected and parsed extensions */ - - mbedtls_x509_buf sig_oid; - mbedtls_x509_buf MBEDTLS_PRIVATE(sig); - mbedtls_md_type_t MBEDTLS_PRIVATE(sig_md); /**< Internal representation of the MD algorithm of the signature algorithm, e.g. MBEDTLS_MD_SHA256 */ - mbedtls_pk_type_t MBEDTLS_PRIVATE(sig_pk); /**< Internal representation of the Public Key algorithm of the signature algorithm, e.g. MBEDTLS_PK_RSA */ - void *MBEDTLS_PRIVATE(sig_opts); /**< Signature options to be passed to mbedtls_pk_verify_ext(), e.g. for RSASSA-PSS */ -} -mbedtls_x509_csr; - -/** - * Container for writing a CSR - */ -typedef struct mbedtls_x509write_csr { - mbedtls_pk_context *MBEDTLS_PRIVATE(key); - mbedtls_asn1_named_data *MBEDTLS_PRIVATE(subject); - mbedtls_md_type_t MBEDTLS_PRIVATE(md_alg); - mbedtls_asn1_named_data *MBEDTLS_PRIVATE(extensions); -} -mbedtls_x509write_csr; - -#if defined(MBEDTLS_X509_CSR_PARSE_C) -/** - * \brief Load a Certificate Signing Request (CSR) in DER format - * - * \note Any unsupported requested extensions are silently - * ignored, unless the critical flag is set, in which case - * the CSR is rejected. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param csr CSR context to fill - * \param buf buffer holding the CRL data - * \param buflen size of the buffer - * - * \return 0 if successful, or a specific X509 error code - */ -int mbedtls_x509_csr_parse_der(mbedtls_x509_csr *csr, - const unsigned char *buf, size_t buflen); - -/** - * \brief The type of certificate extension callbacks. - * - * Callbacks of this type are passed to and used by the - * mbedtls_x509_csr_parse_der_with_ext_cb() routine when - * it encounters either an unsupported extension. - * Future versions of the library may invoke the callback - * in other cases, if and when the need arises. - * - * \param p_ctx An opaque context passed to the callback. - * \param csr The CSR being parsed. - * \param oid The OID of the extension. - * \param critical Whether the extension is critical. - * \param p Pointer to the start of the extension value - * (the content of the OCTET STRING). - * \param end End of extension value. - * - * \note The callback must fail and return a negative error code - * if it can not parse or does not support the extension. - * When the callback fails to parse a critical extension - * mbedtls_x509_csr_parse_der_with_ext_cb() also fails. - * When the callback fails to parse a non critical extension - * mbedtls_x509_csr_parse_der_with_ext_cb() simply skips - * the extension and continues parsing. - * - * \return \c 0 on success. - * \return A negative error code on failure. - */ -typedef int (*mbedtls_x509_csr_ext_cb_t)(void *p_ctx, - mbedtls_x509_csr const *csr, - mbedtls_x509_buf const *oid, - int critical, - const unsigned char *p, - const unsigned char *end); - -/** - * \brief Load a Certificate Signing Request (CSR) in DER format - * - * \note Any unsupported requested extensions are silently - * ignored, unless the critical flag is set, in which case - * the result of the callback function decides whether - * CSR is rejected. - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param csr CSR context to fill - * \param buf buffer holding the CRL data - * \param buflen size of the buffer - * \param cb A callback invoked for every unsupported certificate - * extension. - * \param p_ctx An opaque context passed to the callback. - * - * \return 0 if successful, or a specific X509 error code - */ -int mbedtls_x509_csr_parse_der_with_ext_cb(mbedtls_x509_csr *csr, - const unsigned char *buf, size_t buflen, - mbedtls_x509_csr_ext_cb_t cb, - void *p_ctx); - -/** - * \brief Load a Certificate Signing Request (CSR), DER or PEM format - * - * \note See notes for \c mbedtls_x509_csr_parse_der() - * - * \note If #MBEDTLS_USE_PSA_CRYPTO is enabled, the PSA crypto - * subsystem must have been initialized by calling - * psa_crypto_init() before calling this function. - * - * \param csr CSR context to fill - * \param buf buffer holding the CRL data - * \param buflen size of the buffer - * (including the terminating null byte for PEM data) - * - * \return 0 if successful, or a specific X509 or PEM error code - */ -int mbedtls_x509_csr_parse(mbedtls_x509_csr *csr, const unsigned char *buf, size_t buflen); - -#if defined(MBEDTLS_FS_IO) -/** - * \brief Load a Certificate Signing Request (CSR) - * - * \note See notes for \c mbedtls_x509_csr_parse() - * - * \param csr CSR context to fill - * \param path filename to read the CSR from - * - * \return 0 if successful, or a specific X509 or PEM error code - */ -int mbedtls_x509_csr_parse_file(mbedtls_x509_csr *csr, const char *path); -#endif /* MBEDTLS_FS_IO */ - -#if !defined(MBEDTLS_X509_REMOVE_INFO) -/** - * \brief Returns an informational string about the - * CSR. - * - * \param buf Buffer to write to - * \param size Maximum size of buffer - * \param prefix A line prefix - * \param csr The X509 CSR to represent - * - * \return The length of the string written (not including the - * terminated nul byte), or a negative error code. - */ -int mbedtls_x509_csr_info(char *buf, size_t size, const char *prefix, - const mbedtls_x509_csr *csr); -#endif /* !MBEDTLS_X509_REMOVE_INFO */ - -/** - * \brief Initialize a CSR - * - * \param csr CSR to initialize - */ -void mbedtls_x509_csr_init(mbedtls_x509_csr *csr); - -/** - * \brief Unallocate all CSR data - * - * \param csr CSR to free - */ -void mbedtls_x509_csr_free(mbedtls_x509_csr *csr); -#endif /* MBEDTLS_X509_CSR_PARSE_C */ - -/** \} name Structures and functions for X.509 Certificate Signing Requests (CSR) */ - -#if defined(MBEDTLS_X509_CSR_WRITE_C) -/** - * \brief Initialize a CSR context - * - * \param ctx CSR context to initialize - */ -void mbedtls_x509write_csr_init(mbedtls_x509write_csr *ctx); - -/** - * \brief Set the subject name for a CSR - * Subject names should contain a comma-separated list - * of OID types and values: - * e.g. "C=UK,O=ARM,CN=Mbed TLS Server 1" - * - * \param ctx CSR context to use - * \param subject_name subject name to set - * - * \return 0 if subject name was parsed successfully, or - * a specific error code - */ -int mbedtls_x509write_csr_set_subject_name(mbedtls_x509write_csr *ctx, - const char *subject_name); - -/** - * \brief Set the key for a CSR (public key will be included, - * private key used to sign the CSR when writing it) - * - * \param ctx CSR context to use - * \param key Asymmetric key to include - */ -void mbedtls_x509write_csr_set_key(mbedtls_x509write_csr *ctx, mbedtls_pk_context *key); - -/** - * \brief Set the MD algorithm to use for the signature - * (e.g. MBEDTLS_MD_SHA1) - * - * \param ctx CSR context to use - * \param md_alg MD algorithm to use - */ -void mbedtls_x509write_csr_set_md_alg(mbedtls_x509write_csr *ctx, mbedtls_md_type_t md_alg); - -/** - * \brief Set the Key Usage Extension flags - * (e.g. MBEDTLS_X509_KU_DIGITAL_SIGNATURE | MBEDTLS_X509_KU_KEY_CERT_SIGN) - * - * \param ctx CSR context to use - * \param key_usage key usage flags to set - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - * - * \note The decipherOnly flag from the Key Usage - * extension is represented by bit 8 (i.e. - * 0x8000), which cannot typically be represented - * in an unsigned char. Therefore, the flag - * decipherOnly (i.e. - * #MBEDTLS_X509_KU_DECIPHER_ONLY) cannot be set using this - * function. - */ -int mbedtls_x509write_csr_set_key_usage(mbedtls_x509write_csr *ctx, unsigned char key_usage); - -/** - * \brief Set Subject Alternative Name - * - * \param ctx CSR context to use - * \param san_list List of SAN values - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - * - * \note Only "dnsName", "uniformResourceIdentifier" and "otherName", - * as defined in RFC 5280, are supported. - */ -int mbedtls_x509write_csr_set_subject_alternative_name(mbedtls_x509write_csr *ctx, - const mbedtls_x509_san_list *san_list); - -/** - * \brief Set the Netscape Cert Type flags - * (e.g. MBEDTLS_X509_NS_CERT_TYPE_SSL_CLIENT | MBEDTLS_X509_NS_CERT_TYPE_EMAIL) - * - * \param ctx CSR context to use - * \param ns_cert_type Netscape Cert Type flags to set - * - * \return 0 if successful, or MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_csr_set_ns_cert_type(mbedtls_x509write_csr *ctx, - unsigned char ns_cert_type); - -/** - * \brief Generic function to add to or replace an extension in the - * CSR - * - * \param ctx CSR context to use - * \param oid OID of the extension - * \param oid_len length of the OID - * \param critical Set to 1 to mark the extension as critical, 0 otherwise. - * \param val value of the extension OCTET STRING - * \param val_len length of the value data - * - * \return 0 if successful, or a MBEDTLS_ERR_X509_ALLOC_FAILED - */ -int mbedtls_x509write_csr_set_extension(mbedtls_x509write_csr *ctx, - const char *oid, size_t oid_len, - int critical, - const unsigned char *val, size_t val_len); - -/** - * \brief Free the contents of a CSR context - * - * \param ctx CSR context to free - */ -void mbedtls_x509write_csr_free(mbedtls_x509write_csr *ctx); - -/** - * \brief Write a CSR (Certificate Signing Request) to a - * DER structure - * Note: data is written at the end of the buffer! Use the - * return value to determine where you should start - * using the buffer - * - * \param ctx CSR to write away - * \param buf buffer to write to - * \param size size of the buffer - * \param f_rng RNG function. This must not be \c NULL. - * \param p_rng RNG parameter - * - * \return length of data written if successful, or a specific - * error code - * - * \note \p f_rng is used for the signature operation. - */ -int mbedtls_x509write_csr_der(mbedtls_x509write_csr *ctx, unsigned char *buf, size_t size, - mbedtls_f_rng_t *f_rng, - void *p_rng); - -#if defined(MBEDTLS_PEM_WRITE_C) -/** - * \brief Write a CSR (Certificate Signing Request) to a - * PEM string - * - * \param ctx CSR to write away - * \param buf buffer to write to - * \param size size of the buffer - * \param f_rng RNG function. This must not be \c NULL. - * \param p_rng RNG parameter - * - * \return 0 if successful, or a specific error code - * - * \note \p f_rng is used for the signature operation. - */ -int mbedtls_x509write_csr_pem(mbedtls_x509write_csr *ctx, unsigned char *buf, size_t size, - mbedtls_f_rng_t *f_rng, - void *p_rng); -#endif /* MBEDTLS_PEM_WRITE_C */ -#endif /* MBEDTLS_X509_CSR_WRITE_C */ - -/** \} addtogroup x509_module */ - -#ifdef __cplusplus -} -#endif - -#endif /* mbedtls_x509_csr.h */ diff --git a/android/mbedtls/arm64-v8a/include/psa/build_info.h b/android/mbedtls/arm64-v8a/include/psa/build_info.h deleted file mode 100644 index 3ee6cd7b1..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/build_info.h +++ /dev/null @@ -1,20 +0,0 @@ -/** - * \file psa/build_info.h - * - * \brief Build-time PSA configuration info - * - * Include this file if you need to depend on the - * configuration options defined in mbedtls_config.h or MBEDTLS_CONFIG_FILE - * in PSA cryptography core specific files. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_BUILD_INFO_H -#define PSA_CRYPTO_BUILD_INFO_H - -#include "mbedtls/build_info.h" - -#endif /* PSA_CRYPTO_BUILD_INFO_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto.h b/android/mbedtls/arm64-v8a/include/psa/crypto.h deleted file mode 100644 index 2fe9f35ec..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto.h +++ /dev/null @@ -1,4977 +0,0 @@ -/** - * \file psa/crypto.h - * \brief Platform Security Architecture cryptography module - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_H -#define PSA_CRYPTO_H - -#if defined(MBEDTLS_PSA_CRYPTO_PLATFORM_FILE) -#include MBEDTLS_PSA_CRYPTO_PLATFORM_FILE -#else -#include "crypto_platform.h" -#endif - -#include - -#ifdef __DOXYGEN_ONLY__ -/* This __DOXYGEN_ONLY__ block contains mock definitions for things that - * must be defined in the crypto_platform.h header. These mock definitions - * are present in this file as a convenience to generate pretty-printed - * documentation that includes those definitions. */ - -/** \defgroup platform Implementation-specific definitions - * @{ - */ - -/**@}*/ -#endif /* __DOXYGEN_ONLY__ */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* The file "crypto_types.h" declares types that encode errors, - * algorithms, key types, policies, etc. */ -#include "crypto_types.h" - -/** \defgroup version API version - * @{ - */ - -/** - * The major version of this implementation of the PSA Crypto API - */ -#define PSA_CRYPTO_API_VERSION_MAJOR 1 - -/** - * The minor version of this implementation of the PSA Crypto API - */ -#define PSA_CRYPTO_API_VERSION_MINOR 0 - -/**@}*/ - -/* The file "crypto_values.h" declares macros to build and analyze values - * of integral types defined in "crypto_types.h". */ -#include "crypto_values.h" - -/* The file "crypto_sizes.h" contains definitions for size calculation - * macros whose definitions are implementation-specific. */ -#include "crypto_sizes.h" - -/* The file "crypto_struct.h" contains definitions for - * implementation-specific structs that are declared above. */ -#if defined(MBEDTLS_PSA_CRYPTO_STRUCT_FILE) -#include MBEDTLS_PSA_CRYPTO_STRUCT_FILE -#else -#include "crypto_struct.h" -#endif - -/** \defgroup initialization Library initialization - * @{ - */ - -/** - * \brief Library initialization. - * - * Applications must call this function before calling any other - * function in this module. - * - * Applications may call this function more than once. Once a call - * succeeds, subsequent calls are guaranteed to succeed. - * - * If the application calls other functions before calling psa_crypto_init(), - * the behavior is undefined. Implementations are encouraged to either perform - * the operation as if the library had been initialized or to return - * #PSA_ERROR_BAD_STATE or some other applicable error. In particular, - * implementations should not return a success status if the lack of - * initialization may have security implications, for example due to improper - * seeding of the random number generator. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - */ -psa_status_t psa_crypto_init(void); - -/**@}*/ - -/** \addtogroup attributes - * @{ - */ - -/** \def PSA_KEY_ATTRIBUTES_INIT - * - * This macro returns a suitable initializer for a key attribute structure - * of type #psa_key_attributes_t. - */ - -/** Return an initial value for a key attributes structure. - */ -static psa_key_attributes_t psa_key_attributes_init(void); - -/** Declare a key as persistent and set its key identifier. - * - * If the attribute structure currently declares the key as volatile (which - * is the default content of an attribute structure), this function sets - * the lifetime attribute to #PSA_KEY_LIFETIME_PERSISTENT. - * - * This function does not access storage, it merely stores the given - * value in the structure. - * The persistent key will be written to storage when the attribute - * structure is passed to a key creation function such as - * psa_import_key(), psa_generate_key(), psa_generate_key_custom(), - * psa_key_derivation_output_key(), psa_key_derivation_output_key_custom() - * or psa_copy_key(). - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param key The persistent identifier for the key. - * This can be any value in the range from - * #PSA_KEY_ID_USER_MIN to #PSA_KEY_ID_USER_MAX - * inclusive. - */ -static void psa_set_key_id(psa_key_attributes_t *attributes, - mbedtls_svc_key_id_t key); - -#ifdef MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER -/** Set the owner identifier of a key. - * - * When key identifiers encode key owner identifiers, psa_set_key_id() does - * not allow to define in key attributes the owner of volatile keys as - * psa_set_key_id() enforces the key to be persistent. - * - * This function allows to set in key attributes the owner identifier of a - * key. It is intended to be used for volatile keys. For persistent keys, - * it is recommended to use the PSA Cryptography API psa_set_key_id() to define - * the owner of a key. - * - * \param[out] attributes The attribute structure to write to. - * \param owner The key owner identifier. - */ -static void mbedtls_set_key_owner_id(psa_key_attributes_t *attributes, - mbedtls_key_owner_id_t owner); -#endif - -/** Set the location of a persistent key. - * - * To make a key persistent, you must give it a persistent key identifier - * with psa_set_key_id(). By default, a key that has a persistent identifier - * is stored in the default storage area identifier by - * #PSA_KEY_LIFETIME_PERSISTENT. Call this function to choose a storage - * area, or to explicitly declare the key as volatile. - * - * This function does not access storage, it merely stores the given - * value in the structure. - * The persistent key will be written to storage when the attribute - * structure is passed to a key creation function such as - * psa_import_key(), psa_generate_key(), psa_generate_key_custom(), - * psa_key_derivation_output_key(), psa_key_derivation_output_key_custom() - * or psa_copy_key(). - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param lifetime The lifetime for the key. - * If this is #PSA_KEY_LIFETIME_VOLATILE, the - * key will be volatile, and the key identifier - * attribute is reset to 0. - */ -static void psa_set_key_lifetime(psa_key_attributes_t *attributes, - psa_key_lifetime_t lifetime); - -/** Retrieve the key identifier from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The persistent identifier stored in the attribute structure. - * This value is unspecified if the attribute structure declares - * the key as volatile. - */ -static mbedtls_svc_key_id_t psa_get_key_id( - const psa_key_attributes_t *attributes); - -/** Retrieve the lifetime from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The lifetime value stored in the attribute structure. - */ -static psa_key_lifetime_t psa_get_key_lifetime( - const psa_key_attributes_t *attributes); - -/** Declare usage flags for a key. - * - * Usage flags are part of a key's usage policy. They encode what - * kind of operations are permitted on the key. For more details, - * refer to the documentation of the type #psa_key_usage_t. - * - * This function overwrites any usage flags - * previously set in \p attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param usage_flags The usage flags to write. - */ -static void psa_set_key_usage_flags(psa_key_attributes_t *attributes, - psa_key_usage_t usage_flags); - -/** Retrieve the usage flags from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The usage flags stored in the attribute structure. - */ -static psa_key_usage_t psa_get_key_usage_flags( - const psa_key_attributes_t *attributes); - -/** Declare the permitted algorithm policy for a key. - * - * The permitted algorithm policy of a key encodes which algorithm or - * algorithms are permitted to be used with this key. The following - * algorithm policies are supported: - * - 0 does not allow any cryptographic operation with the key. The key - * may be used for non-cryptographic actions such as exporting (if - * permitted by the usage flags). - * - An algorithm value permits this particular algorithm. - * - An algorithm wildcard built from #PSA_ALG_ANY_HASH allows the specified - * signature scheme with any hash algorithm. - * - An algorithm built from #PSA_ALG_AT_LEAST_THIS_LENGTH_MAC allows - * any MAC algorithm from the same base class (e.g. CMAC) which - * generates/verifies a MAC length greater than or equal to the length - * encoded in the wildcard algorithm. - * - An algorithm built from #PSA_ALG_AEAD_WITH_AT_LEAST_THIS_LENGTH_TAG - * allows any AEAD algorithm from the same base class (e.g. CCM) which - * generates/verifies a tag length greater than or equal to the length - * encoded in the wildcard algorithm. - * - * This function overwrites any algorithm policy - * previously set in \p attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param alg The permitted algorithm policy to write. - */ -static void psa_set_key_algorithm(psa_key_attributes_t *attributes, - psa_algorithm_t alg); - - -/** Retrieve the algorithm policy from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The algorithm stored in the attribute structure. - */ -static psa_algorithm_t psa_get_key_algorithm( - const psa_key_attributes_t *attributes); - -/** Declare the type of a key. - * - * This function overwrites any key type - * previously set in \p attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param type The key type to write. - * If this is 0, the key type in \p attributes - * becomes unspecified. - */ -static void psa_set_key_type(psa_key_attributes_t *attributes, - psa_key_type_t type); - - -/** Declare the size of a key. - * - * This function overwrites any key size previously set in \p attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate each of its arguments exactly once. - * - * \param[out] attributes The attribute structure to write to. - * \param bits The key size in bits. - * If this is 0, the key size in \p attributes - * becomes unspecified. Keys of size 0 are - * not supported. - */ -static void psa_set_key_bits(psa_key_attributes_t *attributes, - size_t bits); - -/** Retrieve the key type from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The key type stored in the attribute structure. - */ -static psa_key_type_t psa_get_key_type(const psa_key_attributes_t *attributes); - -/** Retrieve the key size from key attributes. - * - * This function may be declared as `static` (i.e. without external - * linkage). This function may be provided as a function-like macro, - * but in this case it must evaluate its argument exactly once. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The key size stored in the attribute structure, in bits. - */ -static size_t psa_get_key_bits(const psa_key_attributes_t *attributes); - -/** Retrieve the attributes of a key. - * - * This function first resets the attribute structure as with - * psa_reset_key_attributes(). It then copies the attributes of - * the given key into the given attribute structure. - * - * \note This function may allocate memory or other resources. - * Once you have called this function on an attribute structure, - * you must call psa_reset_key_attributes() to free these resources. - * - * \param[in] key Identifier of the key to query. - * \param[in,out] attributes On success, the attributes of the key. - * On failure, equivalent to a - * freshly-initialized structure. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_get_key_attributes(mbedtls_svc_key_id_t key, - psa_key_attributes_t *attributes); - -/** Reset a key attribute structure to a freshly initialized state. - * - * You must initialize the attribute structure as described in the - * documentation of the type #psa_key_attributes_t before calling this - * function. Once the structure has been initialized, you may call this - * function at any time. - * - * This function frees any auxiliary resources that the structure - * may contain. - * - * \param[in,out] attributes The attribute structure to reset. - */ -void psa_reset_key_attributes(psa_key_attributes_t *attributes); - -/**@}*/ - -/** \defgroup key_management Key management - * @{ - */ - -/** Remove non-essential copies of key material from memory. - * - * If the key identifier designates a volatile key, this functions does not do - * anything and returns successfully. - * - * If the key identifier designates a persistent key, then this function will - * free all resources associated with the key in volatile memory. The key - * data in persistent storage is not affected and the key can still be used. - * - * \param key Identifier of the key to purge. - * - * \retval #PSA_SUCCESS - * The key material will have been removed from memory if it is not - * currently required. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not a valid key identifier. - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_purge_key(mbedtls_svc_key_id_t key); - -/** Make a copy of a key. - * - * Copy key material from one location to another. - * - * This function is primarily useful to copy a key from one location - * to another, since it populates a key using the material from - * another key which may have a different lifetime. - * - * This function may be used to share a key with a different party, - * subject to implementation-defined restrictions on key sharing. - * - * The policy on the source key must have the usage flag - * #PSA_KEY_USAGE_COPY set. - * This flag is sufficient to permit the copy if the key has the lifetime - * #PSA_KEY_LIFETIME_VOLATILE or #PSA_KEY_LIFETIME_PERSISTENT. - * Some secure elements do not provide a way to copy a key without - * making it extractable from the secure element. If a key is located - * in such a secure element, then the key must have both usage flags - * #PSA_KEY_USAGE_COPY and #PSA_KEY_USAGE_EXPORT in order to make - * a copy of the key outside the secure element. - * - * The resulting key may only be used in a way that conforms to - * both the policy of the original key and the policy specified in - * the \p attributes parameter: - * - The usage flags on the resulting key are the bitwise-and of the - * usage flags on the source policy and the usage flags in \p attributes. - * - If both allow the same algorithm or wildcard-based - * algorithm policy, the resulting key has the same algorithm policy. - * - If either of the policies allows an algorithm and the other policy - * allows a wildcard-based algorithm policy that includes this algorithm, - * the resulting key allows the same algorithm. - * - If the policies do not allow any algorithm in common, this function - * fails with the status #PSA_ERROR_INVALID_ARGUMENT. - * - * The effect of this function on implementation-defined attributes is - * implementation-defined. - * - * \param source_key The key to copy. It must allow the usage - * #PSA_KEY_USAGE_COPY. If a private or secret key is - * being copied outside of a secure element it must - * also allow #PSA_KEY_USAGE_EXPORT. - * \param[in] attributes The attributes for the new key. - * They are used as follows: - * - The key type and size may be 0. If either is - * nonzero, it must match the corresponding - * attribute of the source key. - * - The key location (the lifetime and, for - * persistent keys, the key identifier) is - * used directly. - * - The policy constraints (usage flags and - * algorithm policy) are combined from - * the source key and \p attributes so that - * both sets of restrictions apply, as - * described in the documentation of this function. - * \param[out] target_key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE - * \p source_key is invalid. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The lifetime or identifier in \p attributes are invalid, or - * the policy constraints on the source and specified in - * \p attributes are incompatible, or - * \p attributes specifies a key type or key size - * which does not match the attributes of the source key. - * \retval #PSA_ERROR_NOT_PERMITTED - * The source key does not have the #PSA_KEY_USAGE_COPY usage flag, or - * the source key is not exportable and its lifetime does not - * allow copying it to the target's lifetime. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_copy_key(mbedtls_svc_key_id_t source_key, - const psa_key_attributes_t *attributes, - mbedtls_svc_key_id_t *target_key); - - -/** - * \brief Destroy a key. - * - * This function destroys a key from both volatile - * memory and, if applicable, non-volatile storage. Implementations shall - * make a best effort to ensure that the key material cannot be recovered. - * - * This function also erases any metadata such as policies and frees - * resources associated with the key. - * - * If a key is currently in use in a multipart operation, then destroying the - * key will cause the multipart operation to fail. - * - * \warning We can only guarantee that the the key material will - * eventually be wiped from memory. With threading enabled - * and during concurrent execution, copies of the key material may - * still exist until all threads have finished using the key. - * - * \param key Identifier of the key to erase. If this is \c 0, do nothing and - * return #PSA_SUCCESS. - * - * \retval #PSA_SUCCESS - * \p key was a valid identifier and the key material that it - * referred to has been erased. Alternatively, \p key is \c 0. - * \retval #PSA_ERROR_NOT_PERMITTED - * The key cannot be erased because it is - * read-only, either due to a policy or due to physical restrictions. - * \retval #PSA_ERROR_INVALID_HANDLE - * \p key is not a valid identifier nor \c 0. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE - * There was a failure in communication with the cryptoprocessor. - * The key material may still be present in the cryptoprocessor. - * \retval #PSA_ERROR_DATA_INVALID - * This error is typically a result of either storage corruption on a - * cleartext storage backend, or an attempt to read data that was - * written by an incompatible version of the library. - * \retval #PSA_ERROR_STORAGE_FAILURE - * The storage is corrupted. Implementations shall make a best effort - * to erase key material even in this stage, however applications - * should be aware that it may be impossible to guarantee that the - * key material is not recoverable in such cases. - * \retval #PSA_ERROR_CORRUPTION_DETECTED - * An unexpected condition which is not a storage corruption or - * a communication failure occurred. The cryptoprocessor may have - * been compromised. - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_destroy_key(mbedtls_svc_key_id_t key); - -/**@}*/ - -/** \defgroup import_export Key import and export - * @{ - */ - -/** - * \brief Import a key in binary format. - * - * This function supports any output from psa_export_key(). Refer to the - * documentation of psa_export_public_key() for the format of public keys - * and to the documentation of psa_export_key() for the format for - * other key types. - * - * The key data determines the key size. The attributes may optionally - * specify a key size; in this case it must match the size determined - * from the key data. A key size of 0 in \p attributes indicates that - * the key size is solely determined by the key data. - * - * Implementations must reject an attempt to import a key of size 0. - * - * This specification supports a single format for each key type. - * Implementations may support other formats as long as the standard - * format is supported. Implementations that support other formats - * should ensure that the formats are clearly unambiguous so as to - * minimize the risk that an invalid input is accidentally interpreted - * according to a different format. - * - * \param[in] attributes The attributes for the new key. - * The key size is always determined from the - * \p data buffer. - * If the key size in \p attributes is nonzero, - * it must be equal to the size from \p data. - * \param[out] key On success, an identifier to the newly created key. - * For persistent keys, this is the key identifier - * defined in \p attributes. - * \c 0 on failure. - * \param[in] data Buffer containing the key data. The content of this - * buffer is interpreted according to the type declared - * in \p attributes. - * All implementations must support at least the format - * described in the documentation - * of psa_export_key() or psa_export_public_key() for - * the chosen type. Implementations may allow other - * formats, but should be conservative: implementations - * should err on the side of rejecting content if it - * may be erroneous (e.g. wrong type or truncated data). - * \param data_length Size of the \p data buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The key type or key size is not supported, either by the - * implementation in general or in this particular persistent location. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The key attributes, as a whole, are invalid, or - * the key data is not correctly formatted, or - * the size in \p attributes is nonzero and does not match the size - * of the key data. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_import_key(const psa_key_attributes_t *attributes, - const uint8_t *data, - size_t data_length, - mbedtls_svc_key_id_t *key); - - - -/** - * \brief Export a key in binary format. - * - * The output of this function can be passed to psa_import_key() to - * create an equivalent object. - * - * If the implementation of psa_import_key() supports other formats - * beyond the format specified here, the output from psa_export_key() - * must use the representation specified here, not the original - * representation. - * - * For standard key types, the output format is as follows: - * - * - For symmetric keys (including MAC keys), the format is the - * raw bytes of the key. - * - For DES, the key data consists of 8 bytes. The parity bits must be - * correct. - * - For Triple-DES, the format is the concatenation of the - * two or three DES keys. - * - For RSA key pairs (#PSA_KEY_TYPE_RSA_KEY_PAIR), the format - * is the non-encrypted DER encoding of the representation defined by - * PKCS\#1 (RFC 8017) as `RSAPrivateKey`, version 0. - * ``` - * RSAPrivateKey ::= SEQUENCE { - * version INTEGER, -- must be 0 - * modulus INTEGER, -- n - * publicExponent INTEGER, -- e - * privateExponent INTEGER, -- d - * prime1 INTEGER, -- p - * prime2 INTEGER, -- q - * exponent1 INTEGER, -- d mod (p-1) - * exponent2 INTEGER, -- d mod (q-1) - * coefficient INTEGER, -- (inverse of q) mod p - * } - * ``` - * - For elliptic curve key pairs (key types for which - * #PSA_KEY_TYPE_IS_ECC_KEY_PAIR is true), the format is - * a representation of the private value as a `ceiling(m/8)`-byte string - * where `m` is the bit size associated with the curve, i.e. the bit size - * of the order of the curve's coordinate field. This byte string is - * in little-endian order for Montgomery curves (curve types - * `PSA_ECC_FAMILY_CURVEXXX`), and in big-endian order for Weierstrass - * curves (curve types `PSA_ECC_FAMILY_SECTXXX`, `PSA_ECC_FAMILY_SECPXXX` - * and `PSA_ECC_FAMILY_BRAINPOOL_PXXX`). - * For Weierstrass curves, this is the content of the `privateKey` field of - * the `ECPrivateKey` format defined by RFC 5915. For Montgomery curves, - * the format is defined by RFC 7748, and output is masked according to §5. - * For twisted Edwards curves, the private key is as defined by RFC 8032 - * (a 32-byte string for Edwards25519, a 57-byte string for Edwards448). - * - For Diffie-Hellman key exchange key pairs (key types for which - * #PSA_KEY_TYPE_IS_DH_KEY_PAIR is true), the - * format is the representation of the private key `x` as a big-endian byte - * string. The length of the byte string is the private key size in bytes - * (leading zeroes are not stripped). - * - For public keys (key types for which #PSA_KEY_TYPE_IS_PUBLIC_KEY is - * true), the format is the same as for psa_export_public_key(). - * - * The policy on the key must have the usage flag #PSA_KEY_USAGE_EXPORT set. - * - * \param key Identifier of the key to export. It must allow the - * usage #PSA_KEY_USAGE_EXPORT, unless it is a public - * key. - * \param[out] data Buffer where the key data is to be written. - * \param data_size Size of the \p data buffer in bytes. - * \param[out] data_length On success, the number of bytes - * that make up the key data. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_EXPORT flag. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p data buffer is too small. You can determine a - * sufficient buffer size by calling - * #PSA_EXPORT_KEY_OUTPUT_SIZE(\c type, \c bits) - * where \c type is the key type - * and \c bits is the key size in bits. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_export_key(mbedtls_svc_key_id_t key, - uint8_t *data, - size_t data_size, - size_t *data_length); - -/** - * \brief Export a public key or the public part of a key pair in binary format. - * - * The output of this function can be passed to psa_import_key() to - * create an object that is equivalent to the public key. - * - * This specification supports a single format for each key type. - * Implementations may support other formats as long as the standard - * format is supported. Implementations that support other formats - * should ensure that the formats are clearly unambiguous so as to - * minimize the risk that an invalid input is accidentally interpreted - * according to a different format. - * - * For standard key types, the output format is as follows: - * - For RSA public keys (#PSA_KEY_TYPE_RSA_PUBLIC_KEY), the DER encoding of - * the representation defined by RFC 3279 §2.3.1 as `RSAPublicKey`. - * ``` - * RSAPublicKey ::= SEQUENCE { - * modulus INTEGER, -- n - * publicExponent INTEGER } -- e - * ``` - * - For elliptic curve keys on a twisted Edwards curve (key types for which - * #PSA_KEY_TYPE_IS_ECC_PUBLIC_KEY is true and #PSA_KEY_TYPE_ECC_GET_FAMILY - * returns #PSA_ECC_FAMILY_TWISTED_EDWARDS), the public key is as defined - * by RFC 8032 - * (a 32-byte string for Edwards25519, a 57-byte string for Edwards448). - * - For other elliptic curve public keys (key types for which - * #PSA_KEY_TYPE_IS_ECC_PUBLIC_KEY is true), the format is the uncompressed - * representation defined by SEC1 §2.3.3 as the content of an ECPoint. - * Let `m` be the bit size associated with the curve, i.e. the bit size of - * `q` for a curve over `F_q`. The representation consists of: - * - The byte 0x04; - * - `x_P` as a `ceiling(m/8)`-byte string, big-endian; - * - `y_P` as a `ceiling(m/8)`-byte string, big-endian. - * - For Diffie-Hellman key exchange public keys (key types for which - * #PSA_KEY_TYPE_IS_DH_PUBLIC_KEY is true), - * the format is the representation of the public key `y = g^x mod p` as a - * big-endian byte string. The length of the byte string is the length of the - * base prime `p` in bytes. - * - * Exporting a public key object or the public part of a key pair is - * always permitted, regardless of the key's usage flags. - * - * \param key Identifier of the key to export. - * \param[out] data Buffer where the key data is to be written. - * \param data_size Size of the \p data buffer in bytes. - * \param[out] data_length On success, the number of bytes - * that make up the key data. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The key is neither a public key nor a key pair. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p data buffer is too small. You can determine a - * sufficient buffer size by calling - * #PSA_EXPORT_KEY_OUTPUT_SIZE(#PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(\c type), \c bits) - * where \c type is the key type - * and \c bits is the key size in bits. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_export_public_key(mbedtls_svc_key_id_t key, - uint8_t *data, - size_t data_size, - size_t *data_length); - - - -/**@}*/ - -/** \defgroup hash Message digests - * @{ - */ - -/** Calculate the hash (digest) of a message. - * - * \note To verify the hash of a message against an - * expected value, use psa_hash_compare() instead. - * - * \param alg The hash algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_HASH(\p alg) is true). - * \param[in] input Buffer containing the message to hash. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] hash Buffer where the hash is to be written. - * \param hash_size Size of the \p hash buffer in bytes. - * \param[out] hash_length On success, the number of bytes - * that make up the hash value. This is always - * #PSA_HASH_LENGTH(\p alg). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a hash algorithm. - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * \p hash_size is too small - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_compute(psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - uint8_t *hash, - size_t hash_size, - size_t *hash_length); - -/** Calculate the hash (digest) of a message and compare it with a - * reference value. - * - * \param alg The hash algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_HASH(\p alg) is true). - * \param[in] input Buffer containing the message to hash. - * \param input_length Size of the \p input buffer in bytes. - * \param[in] hash Buffer containing the expected hash value. - * \param hash_length Size of the \p hash buffer in bytes. - * - * \retval #PSA_SUCCESS - * The expected hash is identical to the actual hash of the input. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The hash of the message was calculated successfully, but it - * differs from the expected hash. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a hash algorithm. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p input_length or \p hash_length do not match the hash size for \p alg - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_compare(psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - const uint8_t *hash, - size_t hash_length); - -/** The type of the state data structure for multipart hash operations. - * - * Before calling any function on a hash operation object, the application must - * initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_hash_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_hash_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_HASH_OPERATION_INIT, - * for example: - * \code - * psa_hash_operation_t operation = PSA_HASH_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_hash_operation_init() - * to the structure, for example: - * \code - * psa_hash_operation_t operation; - * operation = psa_hash_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_hash_operation_s psa_hash_operation_t; - -/** \def PSA_HASH_OPERATION_INIT - * - * This macro returns a suitable initializer for a hash operation object - * of type #psa_hash_operation_t. - */ - -/** Return an initial value for a hash operation object. - */ -static psa_hash_operation_t psa_hash_operation_init(void); - -/** Set up a multipart hash operation. - * - * The sequence of operations to calculate a hash (message digest) - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_hash_operation_t, e.g. #PSA_HASH_OPERATION_INIT. - * -# Call psa_hash_setup() to specify the algorithm. - * -# Call psa_hash_update() zero, one or more times, passing a fragment - * of the message each time. The hash that is calculated is the hash - * of the concatenation of these messages in order. - * -# To calculate the hash, call psa_hash_finish(). - * To compare the hash with an expected value, call psa_hash_verify(). - * - * If an error occurs at any step after a call to psa_hash_setup(), the - * operation will need to be reset by a call to psa_hash_abort(). The - * application may call psa_hash_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_hash_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A successful call to psa_hash_finish() or psa_hash_verify(). - * - A call to psa_hash_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_hash_operation_t and not yet in use. - * \param alg The hash algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_HASH(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not a supported hash algorithm. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p alg is not a hash algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_setup(psa_hash_operation_t *operation, - psa_algorithm_t alg); - -/** Add a message fragment to a multipart hash operation. - * - * The application must call psa_hash_setup() before calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_hash_abort(). - * - * \param[in,out] operation Active hash operation. - * \param[in] input Buffer containing the message fragment to hash. - * \param input_length Size of the \p input buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_update(psa_hash_operation_t *operation, - const uint8_t *input, - size_t input_length); - -/** Finish the calculation of the hash of a message. - * - * The application must call psa_hash_setup() before calling this function. - * This function calculates the hash of the message formed by concatenating - * the inputs passed to preceding calls to psa_hash_update(). - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_hash_abort(). - * - * \warning Applications should not call this function if they expect - * a specific value for the hash. Call psa_hash_verify() instead. - * Beware that comparing integrity or authenticity data such as - * hash values with a function such as \c memcmp is risky - * because the time taken by the comparison may leak information - * about the hashed data which could allow an attacker to guess - * a valid hash and thereby bypass security controls. - * - * \param[in,out] operation Active hash operation. - * \param[out] hash Buffer where the hash is to be written. - * \param hash_size Size of the \p hash buffer in bytes. - * \param[out] hash_length On success, the number of bytes - * that make up the hash value. This is always - * #PSA_HASH_LENGTH(\c alg) where \c alg is the - * hash algorithm that is calculated. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p hash buffer is too small. You can determine a - * sufficient buffer size by calling #PSA_HASH_LENGTH(\c alg) - * where \c alg is the hash algorithm that is calculated. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_finish(psa_hash_operation_t *operation, - uint8_t *hash, - size_t hash_size, - size_t *hash_length); - -/** Finish the calculation of the hash of a message and compare it with - * an expected value. - * - * The application must call psa_hash_setup() before calling this function. - * This function calculates the hash of the message formed by concatenating - * the inputs passed to preceding calls to psa_hash_update(). It then - * compares the calculated hash with the expected hash passed as a - * parameter to this function. - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_hash_abort(). - * - * \note Implementations shall make the best effort to ensure that the - * comparison between the actual hash and the expected hash is performed - * in constant time. - * - * \param[in,out] operation Active hash operation. - * \param[in] hash Buffer containing the expected hash value. - * \param hash_length Size of the \p hash buffer in bytes. - * - * \retval #PSA_SUCCESS - * The expected hash is identical to the actual hash of the message. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The hash of the message was calculated successfully, but it - * differs from the expected hash. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_verify(psa_hash_operation_t *operation, - const uint8_t *hash, - size_t hash_length); - -/** Abort a hash operation. - * - * Aborting an operation frees all associated resources except for the - * \p operation structure itself. Once aborted, the operation object - * can be reused for another operation by calling - * psa_hash_setup() again. - * - * You may call this function any time after the operation object has - * been initialized by one of the methods described in #psa_hash_operation_t. - * - * In particular, calling psa_hash_abort() after the operation has been - * terminated by a call to psa_hash_abort(), psa_hash_finish() or - * psa_hash_verify() is safe and has no effect. - * - * \param[in,out] operation Initialized hash operation. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_abort(psa_hash_operation_t *operation); - -/** Clone a hash operation. - * - * This function copies the state of an ongoing hash operation to - * a new operation object. In other words, this function is equivalent - * to calling psa_hash_setup() on \p target_operation with the same - * algorithm that \p source_operation was set up for, then - * psa_hash_update() on \p target_operation with the same input that - * that was passed to \p source_operation. After this function returns, the - * two objects are independent, i.e. subsequent calls involving one of - * the objects do not affect the other object. - * - * \param[in] source_operation The active hash operation to clone. - * \param[in,out] target_operation The operation object to set up. - * It must be initialized but not active. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The \p source_operation state is not valid (it must be active), or - * the \p target_operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_hash_clone(const psa_hash_operation_t *source_operation, - psa_hash_operation_t *target_operation); - -/**@}*/ - -/** \defgroup MAC Message authentication codes - * @{ - */ - -/** Calculate the MAC (message authentication code) of a message. - * - * \note To verify the MAC of a message against an - * expected value, use psa_mac_verify() instead. - * Beware that comparing integrity or authenticity data such as - * MAC values with a function such as \c memcmp is risky - * because the time taken by the comparison may leak information - * about the MAC value which could allow an attacker to guess - * a valid MAC and thereby bypass security controls. - * - * \param key Identifier of the key to use for the operation. It - * must allow the usage PSA_KEY_USAGE_SIGN_MESSAGE. - * \param alg The MAC algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_MAC(\p alg) is true). - * \param[in] input Buffer containing the input message. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] mac Buffer where the MAC value is to be written. - * \param mac_size Size of the \p mac buffer in bytes. - * \param[out] mac_length On success, the number of bytes - * that make up the MAC value. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a MAC algorithm. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * \p mac_size is too small - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE - * The key could not be retrieved from storage. - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_compute(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - uint8_t *mac, - size_t mac_size, - size_t *mac_length); - -/** Calculate the MAC of a message and compare it with a reference value. - * - * \param key Identifier of the key to use for the operation. It - * must allow the usage PSA_KEY_USAGE_VERIFY_MESSAGE. - * \param alg The MAC algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_MAC(\p alg) is true). - * \param[in] input Buffer containing the input message. - * \param input_length Size of the \p input buffer in bytes. - * \param[in] mac Buffer containing the expected MAC value. - * \param mac_length Size of the \p mac buffer in bytes. - * - * \retval #PSA_SUCCESS - * The expected MAC is identical to the actual MAC of the input. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The MAC of the message was calculated successfully, but it - * differs from the expected value. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a MAC algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE - * The key could not be retrieved from storage. - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_verify(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - const uint8_t *mac, - size_t mac_length); - -/** The type of the state data structure for multipart MAC operations. - * - * Before calling any function on a MAC operation object, the application must - * initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_mac_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_mac_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_MAC_OPERATION_INIT, - * for example: - * \code - * psa_mac_operation_t operation = PSA_MAC_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_mac_operation_init() - * to the structure, for example: - * \code - * psa_mac_operation_t operation; - * operation = psa_mac_operation_init(); - * \endcode - * - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_mac_operation_s psa_mac_operation_t; - -/** \def PSA_MAC_OPERATION_INIT - * - * This macro returns a suitable initializer for a MAC operation object of type - * #psa_mac_operation_t. - */ - -/** Return an initial value for a MAC operation object. - */ -static psa_mac_operation_t psa_mac_operation_init(void); - -/** Set up a multipart MAC calculation operation. - * - * This function sets up the calculation of the MAC - * (message authentication code) of a byte string. - * To verify the MAC of a message against an - * expected value, use psa_mac_verify_setup() instead. - * - * The sequence of operations to calculate a MAC is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_mac_operation_t, e.g. #PSA_MAC_OPERATION_INIT. - * -# Call psa_mac_sign_setup() to specify the algorithm and key. - * -# Call psa_mac_update() zero, one or more times, passing a fragment - * of the message each time. The MAC that is calculated is the MAC - * of the concatenation of these messages in order. - * -# At the end of the message, call psa_mac_sign_finish() to finish - * calculating the MAC value and retrieve it. - * - * If an error occurs at any step after a call to psa_mac_sign_setup(), the - * operation will need to be reset by a call to psa_mac_abort(). The - * application may call psa_mac_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_mac_sign_setup(), the application must - * eventually terminate the operation through one of the following methods: - * - A successful call to psa_mac_sign_finish(). - * - A call to psa_mac_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_mac_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. It - * must remain valid until the operation terminates. - * It must allow the usage PSA_KEY_USAGE_SIGN_MESSAGE. - * \param alg The MAC algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_MAC(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a MAC algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE - * The key could not be retrieved from storage. - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_sign_setup(psa_mac_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Set up a multipart MAC verification operation. - * - * This function sets up the verification of the MAC - * (message authentication code) of a byte string against an expected value. - * - * The sequence of operations to verify a MAC is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_mac_operation_t, e.g. #PSA_MAC_OPERATION_INIT. - * -# Call psa_mac_verify_setup() to specify the algorithm and key. - * -# Call psa_mac_update() zero, one or more times, passing a fragment - * of the message each time. The MAC that is calculated is the MAC - * of the concatenation of these messages in order. - * -# At the end of the message, call psa_mac_verify_finish() to finish - * calculating the actual MAC of the message and verify it against - * the expected value. - * - * If an error occurs at any step after a call to psa_mac_verify_setup(), the - * operation will need to be reset by a call to psa_mac_abort(). The - * application may call psa_mac_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_mac_verify_setup(), the application must - * eventually terminate the operation through one of the following methods: - * - A successful call to psa_mac_verify_finish(). - * - A call to psa_mac_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_mac_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. It - * must remain valid until the operation terminates. - * It must allow the usage - * PSA_KEY_USAGE_VERIFY_MESSAGE. - * \param alg The MAC algorithm to compute (\c PSA_ALG_XXX value - * such that #PSA_ALG_IS_MAC(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c key is not compatible with \c alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \c alg is not supported or is not a MAC algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE - * The key could not be retrieved from storage. - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_verify_setup(psa_mac_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Add a message fragment to a multipart MAC operation. - * - * The application must call psa_mac_sign_setup() or psa_mac_verify_setup() - * before calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_mac_abort(). - * - * \param[in,out] operation Active MAC operation. - * \param[in] input Buffer containing the message fragment to add to - * the MAC calculation. - * \param input_length Size of the \p input buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_update(psa_mac_operation_t *operation, - const uint8_t *input, - size_t input_length); - -/** Finish the calculation of the MAC of a message. - * - * The application must call psa_mac_sign_setup() before calling this function. - * This function calculates the MAC of the message formed by concatenating - * the inputs passed to preceding calls to psa_mac_update(). - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_mac_abort(). - * - * \warning Applications should not call this function if they expect - * a specific value for the MAC. Call psa_mac_verify_finish() instead. - * Beware that comparing integrity or authenticity data such as - * MAC values with a function such as \c memcmp is risky - * because the time taken by the comparison may leak information - * about the MAC value which could allow an attacker to guess - * a valid MAC and thereby bypass security controls. - * - * \param[in,out] operation Active MAC operation. - * \param[out] mac Buffer where the MAC value is to be written. - * \param mac_size Size of the \p mac buffer in bytes. - * \param[out] mac_length On success, the number of bytes - * that make up the MAC value. This is always - * #PSA_MAC_LENGTH(\c key_type, \c key_bits, \c alg) - * where \c key_type and \c key_bits are the type and - * bit-size respectively of the key and \c alg is the - * MAC algorithm that is calculated. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p mac buffer is too small. You can determine a - * sufficient buffer size by calling PSA_MAC_LENGTH(). - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active mac sign - * operation), or the library has not been previously initialized - * by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_sign_finish(psa_mac_operation_t *operation, - uint8_t *mac, - size_t mac_size, - size_t *mac_length); - -/** Finish the calculation of the MAC of a message and compare it with - * an expected value. - * - * The application must call psa_mac_verify_setup() before calling this function. - * This function calculates the MAC of the message formed by concatenating - * the inputs passed to preceding calls to psa_mac_update(). It then - * compares the calculated MAC with the expected MAC passed as a - * parameter to this function. - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_mac_abort(). - * - * \note Implementations shall make the best effort to ensure that the - * comparison between the actual MAC and the expected MAC is performed - * in constant time. - * - * \param[in,out] operation Active MAC operation. - * \param[in] mac Buffer containing the expected MAC value. - * \param mac_length Size of the \p mac buffer in bytes. - * - * \retval #PSA_SUCCESS - * The expected MAC is identical to the actual MAC of the message. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The MAC of the message was calculated successfully, but it - * differs from the expected MAC. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active mac verify - * operation), or the library has not been previously initialized - * by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_verify_finish(psa_mac_operation_t *operation, - const uint8_t *mac, - size_t mac_length); - -/** Abort a MAC operation. - * - * Aborting an operation frees all associated resources except for the - * \p operation structure itself. Once aborted, the operation object - * can be reused for another operation by calling - * psa_mac_sign_setup() or psa_mac_verify_setup() again. - * - * You may call this function any time after the operation object has - * been initialized by one of the methods described in #psa_mac_operation_t. - * - * In particular, calling psa_mac_abort() after the operation has been - * terminated by a call to psa_mac_abort(), psa_mac_sign_finish() or - * psa_mac_verify_finish() is safe and has no effect. - * - * \param[in,out] operation Initialized MAC operation. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_mac_abort(psa_mac_operation_t *operation); - -/**@}*/ - -/** \defgroup cipher Symmetric ciphers - * @{ - */ - -/** Encrypt a message using a symmetric cipher. - * - * This function encrypts a message with a random IV (initialization - * vector). Use the multipart operation interface with a - * #psa_cipher_operation_t object to provide other forms of IV. - * - * \param key Identifier of the key to use for the operation. - * It must allow the usage #PSA_KEY_USAGE_ENCRYPT. - * \param alg The cipher algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \param[in] input Buffer containing the message to encrypt. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] output Buffer where the output is to be written. - * The output contains the IV followed by - * the ciphertext proper. - * \param output_size Size of the \p output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a cipher algorithm. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_encrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** Decrypt a message using a symmetric cipher. - * - * This function decrypts a message encrypted with a symmetric cipher. - * - * \param key Identifier of the key to use for the operation. - * It must remain valid until the operation - * terminates. It must allow the usage - * #PSA_KEY_USAGE_DECRYPT. - * \param alg The cipher algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \param[in] input Buffer containing the message to decrypt. - * This consists of the IV followed by the - * ciphertext proper. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] output Buffer where the plaintext is to be written. - * \param output_size Size of the \p output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a cipher algorithm. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_decrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** The type of the state data structure for multipart cipher operations. - * - * Before calling any function on a cipher operation object, the application - * must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_cipher_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_cipher_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_CIPHER_OPERATION_INIT, - * for example: - * \code - * psa_cipher_operation_t operation = PSA_CIPHER_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_cipher_operation_init() - * to the structure, for example: - * \code - * psa_cipher_operation_t operation; - * operation = psa_cipher_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_cipher_operation_s psa_cipher_operation_t; - -/** \def PSA_CIPHER_OPERATION_INIT - * - * This macro returns a suitable initializer for a cipher operation object of - * type #psa_cipher_operation_t. - */ - -/** Return an initial value for a cipher operation object. - */ -static psa_cipher_operation_t psa_cipher_operation_init(void); - -/** Set the key for a multipart symmetric encryption operation. - * - * The sequence of operations to encrypt a message with a symmetric cipher - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_cipher_operation_t, e.g. - * #PSA_CIPHER_OPERATION_INIT. - * -# Call psa_cipher_encrypt_setup() to specify the algorithm and key. - * -# Call either psa_cipher_generate_iv() or psa_cipher_set_iv() to - * generate or set the IV (initialization vector). You should use - * psa_cipher_generate_iv() unless the protocol you are implementing - * requires a specific IV value. - * -# Call psa_cipher_update() zero, one or more times, passing a fragment - * of the message each time. - * -# Call psa_cipher_finish(). - * - * If an error occurs at any step after a call to psa_cipher_encrypt_setup(), - * the operation will need to be reset by a call to psa_cipher_abort(). The - * application may call psa_cipher_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_cipher_encrypt_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A successful call to psa_cipher_finish(). - * - A call to psa_cipher_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_cipher_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. - * It must remain valid until the operation - * terminates. It must allow the usage - * #PSA_KEY_USAGE_ENCRYPT. - * \param alg The cipher algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a cipher algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_encrypt_setup(psa_cipher_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Set the key for a multipart symmetric decryption operation. - * - * The sequence of operations to decrypt a message with a symmetric cipher - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_cipher_operation_t, e.g. - * #PSA_CIPHER_OPERATION_INIT. - * -# Call psa_cipher_decrypt_setup() to specify the algorithm and key. - * -# Call psa_cipher_set_iv() with the IV (initialization vector) for the - * decryption. If the IV is prepended to the ciphertext, you can call - * psa_cipher_update() on a buffer containing the IV followed by the - * beginning of the message. - * -# Call psa_cipher_update() zero, one or more times, passing a fragment - * of the message each time. - * -# Call psa_cipher_finish(). - * - * If an error occurs at any step after a call to psa_cipher_decrypt_setup(), - * the operation will need to be reset by a call to psa_cipher_abort(). The - * application may call psa_cipher_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_cipher_decrypt_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A successful call to psa_cipher_finish(). - * - A call to psa_cipher_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_cipher_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. - * It must remain valid until the operation - * terminates. It must allow the usage - * #PSA_KEY_USAGE_DECRYPT. - * \param alg The cipher algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not a cipher algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_decrypt_setup(psa_cipher_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Generate an IV for a symmetric encryption operation. - * - * This function generates a random IV (initialization vector), nonce - * or initial counter value for the encryption operation as appropriate - * for the chosen algorithm, key type and key size. - * - * The application must call psa_cipher_encrypt_setup() before - * calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_cipher_abort(). - * - * \param[in,out] operation Active cipher operation. - * \param[out] iv Buffer where the generated IV is to be written. - * \param iv_size Size of the \p iv buffer in bytes. - * \param[out] iv_length On success, the number of bytes of the - * generated IV. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p iv buffer is too small. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, with no IV set), - * or the library has not been previously initialized - * by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_generate_iv(psa_cipher_operation_t *operation, - uint8_t *iv, - size_t iv_size, - size_t *iv_length); - -/** Set the IV for a symmetric encryption or decryption operation. - * - * This function sets the IV (initialization vector), nonce - * or initial counter value for the encryption or decryption operation. - * - * The application must call psa_cipher_encrypt_setup() before - * calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_cipher_abort(). - * - * \note When encrypting, applications should use psa_cipher_generate_iv() - * instead of this function, unless implementing a protocol that requires - * a non-random IV. - * - * \param[in,out] operation Active cipher operation. - * \param[in] iv Buffer containing the IV to use. - * \param iv_length Size of the IV in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The size of \p iv is not acceptable for the chosen algorithm, - * or the chosen algorithm does not use an IV. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active cipher - * encrypt operation, with no IV set), or the library has not been - * previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_set_iv(psa_cipher_operation_t *operation, - const uint8_t *iv, - size_t iv_length); - -/** Encrypt or decrypt a message fragment in an active cipher operation. - * - * Before calling this function, you must: - * 1. Call either psa_cipher_encrypt_setup() or psa_cipher_decrypt_setup(). - * The choice of setup function determines whether this function - * encrypts or decrypts its input. - * 2. If the algorithm requires an IV, call psa_cipher_generate_iv() - * (recommended when encrypting) or psa_cipher_set_iv(). - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_cipher_abort(). - * - * \param[in,out] operation Active cipher operation. - * \param[in] input Buffer containing the message fragment to - * encrypt or decrypt. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] output Buffer where the output is to be written. - * \param output_size Size of the \p output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, with an IV set - * if required for the algorithm), or the library has not been - * previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_update(psa_cipher_operation_t *operation, - const uint8_t *input, - size_t input_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** Finish encrypting or decrypting a message in a cipher operation. - * - * The application must call psa_cipher_encrypt_setup() or - * psa_cipher_decrypt_setup() before calling this function. The choice - * of setup function determines whether this function encrypts or - * decrypts its input. - * - * This function finishes the encryption or decryption of the message - * formed by concatenating the inputs passed to preceding calls to - * psa_cipher_update(). - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_cipher_abort(). - * - * \param[in,out] operation Active cipher operation. - * \param[out] output Buffer where the output is to be written. - * \param output_size Size of the \p output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The total input size passed to this operation is not valid for - * this particular algorithm. For example, the algorithm is a based - * on block cipher and requires a whole number of blocks, but the - * total input size is not a multiple of the block size. - * \retval #PSA_ERROR_INVALID_PADDING - * This is a decryption operation for an algorithm that includes - * padding, and the ciphertext does not contain valid padding. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, with an IV set - * if required for the algorithm), or the library has not been - * previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_finish(psa_cipher_operation_t *operation, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** Abort a cipher operation. - * - * Aborting an operation frees all associated resources except for the - * \p operation structure itself. Once aborted, the operation object - * can be reused for another operation by calling - * psa_cipher_encrypt_setup() or psa_cipher_decrypt_setup() again. - * - * You may call this function any time after the operation object has - * been initialized as described in #psa_cipher_operation_t. - * - * In particular, calling psa_cipher_abort() after the operation has been - * terminated by a call to psa_cipher_abort() or psa_cipher_finish() - * is safe and has no effect. - * - * \param[in,out] operation Initialized cipher operation. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_cipher_abort(psa_cipher_operation_t *operation); - -/**@}*/ - -/** \defgroup aead Authenticated encryption with associated data (AEAD) - * @{ - */ - -/** Process an authenticated encryption operation. - * - * \param key Identifier of the key to use for the - * operation. It must allow the usage - * #PSA_KEY_USAGE_ENCRYPT. - * \param alg The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * \param[in] nonce Nonce or IV to use. - * \param nonce_length Size of the \p nonce buffer in bytes. - * \param[in] additional_data Additional data that will be authenticated - * but not encrypted. - * \param additional_data_length Size of \p additional_data in bytes. - * \param[in] plaintext Data that will be authenticated and - * encrypted. - * \param plaintext_length Size of \p plaintext in bytes. - * \param[out] ciphertext Output buffer for the authenticated and - * encrypted data. The additional data is not - * part of this output. For algorithms where the - * encrypted data and the authentication tag - * are defined as separate outputs, the - * authentication tag is appended to the - * encrypted data. - * \param ciphertext_size Size of the \p ciphertext buffer in bytes. - * This must be appropriate for the selected - * algorithm and key: - * - A sufficient output size is - * #PSA_AEAD_ENCRYPT_OUTPUT_SIZE(\c key_type, - * \p alg, \p plaintext_length) where - * \c key_type is the type of \p key. - * - #PSA_AEAD_ENCRYPT_OUTPUT_MAX_SIZE(\p - * plaintext_length) evaluates to the maximum - * ciphertext size of any supported AEAD - * encryption. - * \param[out] ciphertext_length On success, the size of the output - * in the \p ciphertext buffer. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not an AEAD algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * \p ciphertext_size is too small. - * #PSA_AEAD_ENCRYPT_OUTPUT_SIZE(\c key_type, \p alg, - * \p plaintext_length) or - * #PSA_AEAD_ENCRYPT_OUTPUT_MAX_SIZE(\p plaintext_length) can be used to - * determine the required buffer size. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_encrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *nonce, - size_t nonce_length, - const uint8_t *additional_data, - size_t additional_data_length, - const uint8_t *plaintext, - size_t plaintext_length, - uint8_t *ciphertext, - size_t ciphertext_size, - size_t *ciphertext_length); - -/** Process an authenticated decryption operation. - * - * \param key Identifier of the key to use for the - * operation. It must allow the usage - * #PSA_KEY_USAGE_DECRYPT. - * \param alg The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * \param[in] nonce Nonce or IV to use. - * \param nonce_length Size of the \p nonce buffer in bytes. - * \param[in] additional_data Additional data that has been authenticated - * but not encrypted. - * \param additional_data_length Size of \p additional_data in bytes. - * \param[in] ciphertext Data that has been authenticated and - * encrypted. For algorithms where the - * encrypted data and the authentication tag - * are defined as separate inputs, the buffer - * must contain the encrypted data followed - * by the authentication tag. - * \param ciphertext_length Size of \p ciphertext in bytes. - * \param[out] plaintext Output buffer for the decrypted data. - * \param plaintext_size Size of the \p plaintext buffer in bytes. - * This must be appropriate for the selected - * algorithm and key: - * - A sufficient output size is - * #PSA_AEAD_DECRYPT_OUTPUT_SIZE(\c key_type, - * \p alg, \p ciphertext_length) where - * \c key_type is the type of \p key. - * - #PSA_AEAD_DECRYPT_OUTPUT_MAX_SIZE(\p - * ciphertext_length) evaluates to the maximum - * plaintext size of any supported AEAD - * decryption. - * \param[out] plaintext_length On success, the size of the output - * in the \p plaintext buffer. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The ciphertext is not authentic. - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not an AEAD algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * \p plaintext_size is too small. - * #PSA_AEAD_DECRYPT_OUTPUT_SIZE(\c key_type, \p alg, - * \p ciphertext_length) or - * #PSA_AEAD_DECRYPT_OUTPUT_MAX_SIZE(\p ciphertext_length) can be used - * to determine the required buffer size. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_decrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *nonce, - size_t nonce_length, - const uint8_t *additional_data, - size_t additional_data_length, - const uint8_t *ciphertext, - size_t ciphertext_length, - uint8_t *plaintext, - size_t plaintext_size, - size_t *plaintext_length); - -/** The type of the state data structure for multipart AEAD operations. - * - * Before calling any function on an AEAD operation object, the application - * must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_aead_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_aead_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_AEAD_OPERATION_INIT, - * for example: - * \code - * psa_aead_operation_t operation = PSA_AEAD_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_aead_operation_init() - * to the structure, for example: - * \code - * psa_aead_operation_t operation; - * operation = psa_aead_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_aead_operation_s psa_aead_operation_t; - -/** \def PSA_AEAD_OPERATION_INIT - * - * This macro returns a suitable initializer for an AEAD operation object of - * type #psa_aead_operation_t. - */ - -/** Return an initial value for an AEAD operation object. - */ -static psa_aead_operation_t psa_aead_operation_init(void); - -/** Set the key for a multipart authenticated encryption operation. - * - * The sequence of operations to encrypt a message with authentication - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_aead_operation_t, e.g. - * #PSA_AEAD_OPERATION_INIT. - * -# Call psa_aead_encrypt_setup() to specify the algorithm and key. - * -# If needed, call psa_aead_set_lengths() to specify the length of the - * inputs to the subsequent calls to psa_aead_update_ad() and - * psa_aead_update(). See the documentation of psa_aead_set_lengths() - * for details. - * -# Call either psa_aead_generate_nonce() or psa_aead_set_nonce() to - * generate or set the nonce. You should use - * psa_aead_generate_nonce() unless the protocol you are implementing - * requires a specific nonce value. - * -# Call psa_aead_update_ad() zero, one or more times, passing a fragment - * of the non-encrypted additional authenticated data each time. - * -# Call psa_aead_update() zero, one or more times, passing a fragment - * of the message to encrypt each time. - * -# Call psa_aead_finish(). - * - * If an error occurs at any step after a call to psa_aead_encrypt_setup(), - * the operation will need to be reset by a call to psa_aead_abort(). The - * application may call psa_aead_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_aead_encrypt_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A successful call to psa_aead_finish(). - * - A call to psa_aead_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_aead_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. - * It must remain valid until the operation - * terminates. It must allow the usage - * #PSA_KEY_USAGE_ENCRYPT. - * \param alg The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not an AEAD algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_encrypt_setup(psa_aead_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Set the key for a multipart authenticated decryption operation. - * - * The sequence of operations to decrypt a message with authentication - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_aead_operation_t, e.g. - * #PSA_AEAD_OPERATION_INIT. - * -# Call psa_aead_decrypt_setup() to specify the algorithm and key. - * -# If needed, call psa_aead_set_lengths() to specify the length of the - * inputs to the subsequent calls to psa_aead_update_ad() and - * psa_aead_update(). See the documentation of psa_aead_set_lengths() - * for details. - * -# Call psa_aead_set_nonce() with the nonce for the decryption. - * -# Call psa_aead_update_ad() zero, one or more times, passing a fragment - * of the non-encrypted additional authenticated data each time. - * -# Call psa_aead_update() zero, one or more times, passing a fragment - * of the ciphertext to decrypt each time. - * -# Call psa_aead_verify(). - * - * If an error occurs at any step after a call to psa_aead_decrypt_setup(), - * the operation will need to be reset by a call to psa_aead_abort(). The - * application may call psa_aead_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_aead_decrypt_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A successful call to psa_aead_verify(). - * - A call to psa_aead_abort(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized as per the documentation for - * #psa_aead_operation_t and not yet in use. - * \param key Identifier of the key to use for the operation. - * It must remain valid until the operation - * terminates. It must allow the usage - * #PSA_KEY_USAGE_DECRYPT. - * \param alg The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not compatible with \p alg. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not supported or is not an AEAD algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or the - * library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_decrypt_setup(psa_aead_operation_t *operation, - mbedtls_svc_key_id_t key, - psa_algorithm_t alg); - -/** Generate a random nonce for an authenticated encryption operation. - * - * This function generates a random nonce for the authenticated encryption - * operation with an appropriate size for the chosen algorithm, key type - * and key size. - * - * The application must call psa_aead_encrypt_setup() before - * calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \param[in,out] operation Active AEAD operation. - * \param[out] nonce Buffer where the generated nonce is to be - * written. - * \param nonce_size Size of the \p nonce buffer in bytes. - * \param[out] nonce_length On success, the number of bytes of the - * generated nonce. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p nonce buffer is too small. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active aead encrypt - * operation, with no nonce set), or the library has not been - * previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_generate_nonce(psa_aead_operation_t *operation, - uint8_t *nonce, - size_t nonce_size, - size_t *nonce_length); - -/** Set the nonce for an authenticated encryption or decryption operation. - * - * This function sets the nonce for the authenticated - * encryption or decryption operation. - * - * The application must call psa_aead_encrypt_setup() or - * psa_aead_decrypt_setup() before calling this function. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \note When encrypting, applications should use psa_aead_generate_nonce() - * instead of this function, unless implementing a protocol that requires - * a non-random IV. - * - * \param[in,out] operation Active AEAD operation. - * \param[in] nonce Buffer containing the nonce to use. - * \param nonce_length Size of the nonce in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The size of \p nonce is not acceptable for the chosen algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, with no nonce - * set), or the library has not been previously initialized - * by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_set_nonce(psa_aead_operation_t *operation, - const uint8_t *nonce, - size_t nonce_length); - -/** Declare the lengths of the message and additional data for AEAD. - * - * The application must call this function before calling - * psa_aead_update_ad() or psa_aead_update() if the algorithm for - * the operation requires it. If the algorithm does not require it, - * calling this function is optional, but if this function is called - * then the implementation must enforce the lengths. - * - * You may call this function before or after setting the nonce with - * psa_aead_set_nonce() or psa_aead_generate_nonce(). - * - * - For #PSA_ALG_CCM, calling this function is required. - * - For the other AEAD algorithms defined in this specification, calling - * this function is not required. - * - For vendor-defined algorithm, refer to the vendor documentation. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \param[in,out] operation Active AEAD operation. - * \param ad_length Size of the non-encrypted additional - * authenticated data in bytes. - * \param plaintext_length Size of the plaintext to encrypt in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * At least one of the lengths is not acceptable for the chosen - * algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, and - * psa_aead_update_ad() and psa_aead_update() must not have been - * called yet), or the library has not been previously initialized - * by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_set_lengths(psa_aead_operation_t *operation, - size_t ad_length, - size_t plaintext_length); - -/** Pass additional data to an active AEAD operation. - * - * Additional data is authenticated, but not encrypted. - * - * You may call this function multiple times to pass successive fragments - * of the additional data. You may not call this function after passing - * data to encrypt or decrypt with psa_aead_update(). - * - * Before calling this function, you must: - * 1. Call either psa_aead_encrypt_setup() or psa_aead_decrypt_setup(). - * 2. Set the nonce with psa_aead_generate_nonce() or psa_aead_set_nonce(). - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \warning When decrypting, until psa_aead_verify() has returned #PSA_SUCCESS, - * there is no guarantee that the input is valid. Therefore, until - * you have called psa_aead_verify() and it has returned #PSA_SUCCESS, - * treat the input as untrusted and prepare to undo any action that - * depends on the input if psa_aead_verify() returns an error status. - * - * \param[in,out] operation Active AEAD operation. - * \param[in] input Buffer containing the fragment of - * additional data. - * \param input_length Size of the \p input buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The total input length overflows the additional data length that - * was previously specified with psa_aead_set_lengths(). - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, have a nonce - * set, have lengths set if required by the algorithm, and - * psa_aead_update() must not have been called yet), or the library - * has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_update_ad(psa_aead_operation_t *operation, - const uint8_t *input, - size_t input_length); - -/** Encrypt or decrypt a message fragment in an active AEAD operation. - * - * Before calling this function, you must: - * 1. Call either psa_aead_encrypt_setup() or psa_aead_decrypt_setup(). - * The choice of setup function determines whether this function - * encrypts or decrypts its input. - * 2. Set the nonce with psa_aead_generate_nonce() or psa_aead_set_nonce(). - * 3. Call psa_aead_update_ad() to pass all the additional data. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \warning When decrypting, until psa_aead_verify() has returned #PSA_SUCCESS, - * there is no guarantee that the input is valid. Therefore, until - * you have called psa_aead_verify() and it has returned #PSA_SUCCESS: - * - Do not use the output in any way other than storing it in a - * confidential location. If you take any action that depends - * on the tentative decrypted data, this action will need to be - * undone if the input turns out not to be valid. Furthermore, - * if an adversary can observe that this action took place - * (for example through timing), they may be able to use this - * fact as an oracle to decrypt any message encrypted with the - * same key. - * - In particular, do not copy the output anywhere but to a - * memory or storage space that you have exclusive access to. - * - * This function does not require the input to be aligned to any - * particular block boundary. If the implementation can only process - * a whole block at a time, it must consume all the input provided, but - * it may delay the end of the corresponding output until a subsequent - * call to psa_aead_update(), psa_aead_finish() or psa_aead_verify() - * provides sufficient input. The amount of data that can be delayed - * in this way is bounded by #PSA_AEAD_UPDATE_OUTPUT_SIZE. - * - * \param[in,out] operation Active AEAD operation. - * \param[in] input Buffer containing the message fragment to - * encrypt or decrypt. - * \param input_length Size of the \p input buffer in bytes. - * \param[out] output Buffer where the output is to be written. - * \param output_size Size of the \p output buffer in bytes. - * This must be appropriate for the selected - * algorithm and key: - * - A sufficient output size is - * #PSA_AEAD_UPDATE_OUTPUT_SIZE(\c key_type, - * \c alg, \p input_length) where - * \c key_type is the type of key and \c alg is - * the algorithm that were used to set up the - * operation. - * - #PSA_AEAD_UPDATE_OUTPUT_MAX_SIZE(\p - * input_length) evaluates to the maximum - * output size of any supported AEAD - * algorithm. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. - * #PSA_AEAD_UPDATE_OUTPUT_SIZE(\c key_type, \c alg, \p input_length) or - * #PSA_AEAD_UPDATE_OUTPUT_MAX_SIZE(\p input_length) can be used to - * determine the required buffer size. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The total length of input to psa_aead_update_ad() so far is - * less than the additional data length that was previously - * specified with psa_aead_set_lengths(), or - * the total input length overflows the plaintext length that - * was previously specified with psa_aead_set_lengths(). - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, have a nonce - * set, and have lengths set if required by the algorithm), or the - * library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_update(psa_aead_operation_t *operation, - const uint8_t *input, - size_t input_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** Finish encrypting a message in an AEAD operation. - * - * The operation must have been set up with psa_aead_encrypt_setup(). - * - * This function finishes the authentication of the additional data - * formed by concatenating the inputs passed to preceding calls to - * psa_aead_update_ad() with the plaintext formed by concatenating the - * inputs passed to preceding calls to psa_aead_update(). - * - * This function has two output buffers: - * - \p ciphertext contains trailing ciphertext that was buffered from - * preceding calls to psa_aead_update(). - * - \p tag contains the authentication tag. - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \param[in,out] operation Active AEAD operation. - * \param[out] ciphertext Buffer where the last part of the ciphertext - * is to be written. - * \param ciphertext_size Size of the \p ciphertext buffer in bytes. - * This must be appropriate for the selected - * algorithm and key: - * - A sufficient output size is - * #PSA_AEAD_FINISH_OUTPUT_SIZE(\c key_type, - * \c alg) where \c key_type is the type of key - * and \c alg is the algorithm that were used to - * set up the operation. - * - #PSA_AEAD_FINISH_OUTPUT_MAX_SIZE evaluates to - * the maximum output size of any supported AEAD - * algorithm. - * \param[out] ciphertext_length On success, the number of bytes of - * returned ciphertext. - * \param[out] tag Buffer where the authentication tag is - * to be written. - * \param tag_size Size of the \p tag buffer in bytes. - * This must be appropriate for the selected - * algorithm and key: - * - The exact tag size is #PSA_AEAD_TAG_LENGTH(\c - * key_type, \c key_bits, \c alg) where - * \c key_type and \c key_bits are the type and - * bit-size of the key, and \c alg is the - * algorithm that were used in the call to - * psa_aead_encrypt_setup(). - * - #PSA_AEAD_TAG_MAX_SIZE evaluates to the - * maximum tag size of any supported AEAD - * algorithm. - * \param[out] tag_length On success, the number of bytes - * that make up the returned tag. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p ciphertext or \p tag buffer is too small. - * #PSA_AEAD_FINISH_OUTPUT_SIZE(\c key_type, \c alg) or - * #PSA_AEAD_FINISH_OUTPUT_MAX_SIZE can be used to determine the - * required \p ciphertext buffer size. #PSA_AEAD_TAG_LENGTH(\c key_type, - * \c key_bits, \c alg) or #PSA_AEAD_TAG_MAX_SIZE can be used to - * determine the required \p tag buffer size. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The total length of input to psa_aead_update_ad() so far is - * less than the additional data length that was previously - * specified with psa_aead_set_lengths(), or - * the total length of input to psa_aead_update() so far is - * less than the plaintext length that was previously - * specified with psa_aead_set_lengths(). - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active encryption - * operation with a nonce set), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_finish(psa_aead_operation_t *operation, - uint8_t *ciphertext, - size_t ciphertext_size, - size_t *ciphertext_length, - uint8_t *tag, - size_t tag_size, - size_t *tag_length); - -/** Finish authenticating and decrypting a message in an AEAD operation. - * - * The operation must have been set up with psa_aead_decrypt_setup(). - * - * This function finishes the authenticated decryption of the message - * components: - * - * - The additional data consisting of the concatenation of the inputs - * passed to preceding calls to psa_aead_update_ad(). - * - The ciphertext consisting of the concatenation of the inputs passed to - * preceding calls to psa_aead_update(). - * - The tag passed to this function call. - * - * If the authentication tag is correct, this function outputs any remaining - * plaintext and reports success. If the authentication tag is not correct, - * this function returns #PSA_ERROR_INVALID_SIGNATURE. - * - * When this function returns successfully, the operation becomes inactive. - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_aead_abort(). - * - * \note Implementations shall make the best effort to ensure that the - * comparison between the actual tag and the expected tag is performed - * in constant time. - * - * \param[in,out] operation Active AEAD operation. - * \param[out] plaintext Buffer where the last part of the plaintext - * is to be written. This is the remaining data - * from previous calls to psa_aead_update() - * that could not be processed until the end - * of the input. - * \param plaintext_size Size of the \p plaintext buffer in bytes. - * This must be appropriate for the selected algorithm and key: - * - A sufficient output size is - * #PSA_AEAD_VERIFY_OUTPUT_SIZE(\c key_type, - * \c alg) where \c key_type is the type of key - * and \c alg is the algorithm that were used to - * set up the operation. - * - #PSA_AEAD_VERIFY_OUTPUT_MAX_SIZE evaluates to - * the maximum output size of any supported AEAD - * algorithm. - * \param[out] plaintext_length On success, the number of bytes of - * returned plaintext. - * \param[in] tag Buffer containing the authentication tag. - * \param tag_length Size of the \p tag buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The calculations were successful, but the authentication tag is - * not correct. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p plaintext buffer is too small. - * #PSA_AEAD_VERIFY_OUTPUT_SIZE(\c key_type, \c alg) or - * #PSA_AEAD_VERIFY_OUTPUT_MAX_SIZE can be used to determine the - * required buffer size. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The total length of input to psa_aead_update_ad() so far is - * less than the additional data length that was previously - * specified with psa_aead_set_lengths(), or - * the total length of input to psa_aead_update() so far is - * less than the plaintext length that was previously - * specified with psa_aead_set_lengths(). - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be an active decryption - * operation with a nonce set), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_verify(psa_aead_operation_t *operation, - uint8_t *plaintext, - size_t plaintext_size, - size_t *plaintext_length, - const uint8_t *tag, - size_t tag_length); - -/** Abort an AEAD operation. - * - * Aborting an operation frees all associated resources except for the - * \p operation structure itself. Once aborted, the operation object - * can be reused for another operation by calling - * psa_aead_encrypt_setup() or psa_aead_decrypt_setup() again. - * - * You may call this function any time after the operation object has - * been initialized as described in #psa_aead_operation_t. - * - * In particular, calling psa_aead_abort() after the operation has been - * terminated by a call to psa_aead_abort(), psa_aead_finish() or - * psa_aead_verify() is safe and has no effect. - * - * \param[in,out] operation Initialized AEAD operation. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_aead_abort(psa_aead_operation_t *operation); - -/**@}*/ - -/** \defgroup asymmetric Asymmetric cryptography - * @{ - */ - -/** - * \brief Sign a message with a private key. For hash-and-sign algorithms, - * this includes the hashing step. - * - * \note To perform a multi-part hash-and-sign signature algorithm, first use - * a multi-part hash operation and then pass the resulting hash to - * psa_sign_hash(). PSA_ALG_GET_HASH(\p alg) can be used to determine the - * hash algorithm to use. - * - * \param[in] key Identifier of the key to use for the operation. - * It must be an asymmetric key pair. The key must - * allow the usage #PSA_KEY_USAGE_SIGN_MESSAGE. - * \param[in] alg An asymmetric signature algorithm (PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_MESSAGE(\p alg) - * is true), that is compatible with the type of - * \p key. - * \param[in] input The input message to sign. - * \param[in] input_length Size of the \p input buffer in bytes. - * \param[out] signature Buffer where the signature is to be written. - * \param[in] signature_size Size of the \p signature buffer in bytes. This - * must be appropriate for the selected - * algorithm and key: - * - The required signature size is - * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) - * where \c key_type and \c key_bits are the type and - * bit-size respectively of key. - * - #PSA_SIGNATURE_MAX_SIZE evaluates to the - * maximum signature size of any supported - * signature algorithm. - * \param[out] signature_length On success, the number of bytes that make up - * the returned signature value. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_SIGN_MESSAGE flag, - * or it does not permit the requested algorithm. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p signature buffer is too small. You can - * determine a sufficient buffer size by calling - * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) - * where \c key_type and \c key_bits are the type and bit-size - * respectively of \p key. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_sign_message(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - uint8_t *signature, - size_t signature_size, - size_t *signature_length); - -/** \brief Verify the signature of a message with a public key, using - * a hash-and-sign verification algorithm. - * - * \note To perform a multi-part hash-and-sign signature verification - * algorithm, first use a multi-part hash operation to hash the message - * and then pass the resulting hash to psa_verify_hash(). - * PSA_ALG_GET_HASH(\p alg) can be used to determine the hash algorithm - * to use. - * - * \param[in] key Identifier of the key to use for the operation. - * It must be a public key or an asymmetric key - * pair. The key must allow the usage - * #PSA_KEY_USAGE_VERIFY_MESSAGE. - * \param[in] alg An asymmetric signature algorithm (PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_MESSAGE(\p alg) - * is true), that is compatible with the type of - * \p key. - * \param[in] input The message whose signature is to be verified. - * \param[in] input_length Size of the \p input buffer in bytes. - * \param[in] signature Buffer containing the signature to verify. - * \param[in] signature_length Size of the \p signature buffer in bytes. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_SIGN_MESSAGE flag, - * or it does not permit the requested algorithm. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The calculation was performed successfully, but the passed signature - * is not a valid signature. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_verify_message(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - const uint8_t *signature, - size_t signature_length); - -/** - * \brief Sign a hash or short message with a private key. - * - * Note that to perform a hash-and-sign signature algorithm, you must - * first calculate the hash by calling psa_hash_setup(), psa_hash_update() - * and psa_hash_finish(), or alternatively by calling psa_hash_compute(). - * Then pass the resulting hash as the \p hash - * parameter to this function. You can use #PSA_ALG_SIGN_GET_HASH(\p alg) - * to determine the hash algorithm to use. - * - * \param key Identifier of the key to use for the operation. - * It must be an asymmetric key pair. The key must - * allow the usage #PSA_KEY_USAGE_SIGN_HASH. - * \param alg A signature algorithm (PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_HASH(\p alg) - * is true), that is compatible with - * the type of \p key. - * \param[in] hash The hash or message to sign. - * \param hash_length Size of the \p hash buffer in bytes. - * \param[out] signature Buffer where the signature is to be written. - * \param signature_size Size of the \p signature buffer in bytes. - * \param[out] signature_length On success, the number of bytes - * that make up the returned signature value. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p signature buffer is too small. You can - * determine a sufficient buffer size by calling - * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) - * where \c key_type and \c key_bits are the type and bit-size - * respectively of \p key. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_sign_hash(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *hash, - size_t hash_length, - uint8_t *signature, - size_t signature_size, - size_t *signature_length); - -/** - * \brief Verify the signature of a hash or short message using a public key. - * - * Note that to perform a hash-and-sign signature algorithm, you must - * first calculate the hash by calling psa_hash_setup(), psa_hash_update() - * and psa_hash_finish(), or alternatively by calling psa_hash_compute(). - * Then pass the resulting hash as the \p hash - * parameter to this function. You can use #PSA_ALG_SIGN_GET_HASH(\p alg) - * to determine the hash algorithm to use. - * - * \param key Identifier of the key to use for the operation. It - * must be a public key or an asymmetric key pair. The - * key must allow the usage - * #PSA_KEY_USAGE_VERIFY_HASH. - * \param alg A signature algorithm (PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_HASH(\p alg) - * is true), that is compatible with - * the type of \p key. - * \param[in] hash The hash or message whose signature is to be - * verified. - * \param hash_length Size of the \p hash buffer in bytes. - * \param[in] signature Buffer containing the signature to verify. - * \param signature_length Size of the \p signature buffer in bytes. - * - * \retval #PSA_SUCCESS - * The signature is valid. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The calculation was performed successfully, but the passed - * signature is not a valid signature. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_verify_hash(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *hash, - size_t hash_length, - const uint8_t *signature, - size_t signature_length); - -/** - * \brief Encrypt a short message with a public key. - * - * \param key Identifier of the key to use for the operation. - * It must be a public key or an asymmetric key - * pair. It must allow the usage - * #PSA_KEY_USAGE_ENCRYPT. - * \param alg An asymmetric encryption algorithm that is - * compatible with the type of \p key. - * \param[in] input The message to encrypt. - * \param input_length Size of the \p input buffer in bytes. - * \param[in] salt A salt or label, if supported by the - * encryption algorithm. - * If the algorithm does not support a - * salt, pass \c NULL. - * If the algorithm supports an optional - * salt and you do not want to pass a salt, - * pass \c NULL. - * - * - For #PSA_ALG_RSA_PKCS1V15_CRYPT, no salt is - * supported. - * \param salt_length Size of the \p salt buffer in bytes. - * If \p salt is \c NULL, pass 0. - * \param[out] output Buffer where the encrypted message is to - * be written. - * \param output_size Size of the \p output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. You can - * determine a sufficient buffer size by calling - * #PSA_ASYMMETRIC_ENCRYPT_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) - * where \c key_type and \c key_bits are the type and bit-size - * respectively of \p key. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_asymmetric_encrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - const uint8_t *salt, - size_t salt_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** - * \brief Decrypt a short message with a private key. - * - * \param key Identifier of the key to use for the operation. - * It must be an asymmetric key pair. It must - * allow the usage #PSA_KEY_USAGE_DECRYPT. - * \param alg An asymmetric encryption algorithm that is - * compatible with the type of \p key. - * \param[in] input The message to decrypt. - * \param input_length Size of the \p input buffer in bytes. - * \param[in] salt A salt or label, if supported by the - * encryption algorithm. - * If the algorithm does not support a - * salt, pass \c NULL. - * If the algorithm supports an optional - * salt and you do not want to pass a salt, - * pass \c NULL. - * - * - For #PSA_ALG_RSA_PKCS1V15_CRYPT, no salt is - * supported. - * \param salt_length Size of the \p salt buffer in bytes. - * If \p salt is \c NULL, pass 0. - * \param[out] output Buffer where the decrypted message is to - * be written. - * \param output_size Size of the \c output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. You can - * determine a sufficient buffer size by calling - * #PSA_ASYMMETRIC_DECRYPT_OUTPUT_SIZE(\c key_type, \c key_bits, \p alg) - * where \c key_type and \c key_bits are the type and bit-size - * respectively of \p key. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_INVALID_PADDING \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_asymmetric_decrypt(mbedtls_svc_key_id_t key, - psa_algorithm_t alg, - const uint8_t *input, - size_t input_length, - const uint8_t *salt, - size_t salt_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/**@}*/ - -/** \defgroup key_derivation Key derivation and pseudorandom generation - * @{ - */ - -/** The type of the state data structure for key derivation operations. - * - * Before calling any function on a key derivation operation object, the - * application must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_key_derivation_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_key_derivation_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_KEY_DERIVATION_OPERATION_INIT, - * for example: - * \code - * psa_key_derivation_operation_t operation = PSA_KEY_DERIVATION_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_key_derivation_operation_init() - * to the structure, for example: - * \code - * psa_key_derivation_operation_t operation; - * operation = psa_key_derivation_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. - */ -typedef struct psa_key_derivation_s psa_key_derivation_operation_t; - -/** \def PSA_KEY_DERIVATION_OPERATION_INIT - * - * This macro returns a suitable initializer for a key derivation operation - * object of type #psa_key_derivation_operation_t. - */ - -/** Return an initial value for a key derivation operation object. - */ -static psa_key_derivation_operation_t psa_key_derivation_operation_init(void); - -/** Set up a key derivation operation. - * - * A key derivation algorithm takes some inputs and uses them to generate - * a byte stream in a deterministic way. - * This byte stream can be used to produce keys and other - * cryptographic material. - * - * To derive a key: - * -# Start with an initialized object of type #psa_key_derivation_operation_t. - * -# Call psa_key_derivation_setup() to select the algorithm. - * -# Provide the inputs for the key derivation by calling - * psa_key_derivation_input_bytes() or psa_key_derivation_input_key() - * as appropriate. Which inputs are needed, in what order, and whether - * they may be keys and if so of what type depends on the algorithm. - * -# Optionally set the operation's maximum capacity with - * psa_key_derivation_set_capacity(). You may do this before, in the middle - * of or after providing inputs. For some algorithms, this step is mandatory - * because the output depends on the maximum capacity. - * -# To derive a key, call psa_key_derivation_output_key() or - * psa_key_derivation_output_key_custom(). - * To derive a byte string for a different purpose, call - * psa_key_derivation_output_bytes(). - * Successive calls to these functions use successive output bytes - * calculated by the key derivation algorithm. - * -# Clean up the key derivation operation object with - * psa_key_derivation_abort(). - * - * If this function returns an error, the key derivation operation object is - * not changed. - * - * If an error occurs at any step after a call to psa_key_derivation_setup(), - * the operation will need to be reset by a call to psa_key_derivation_abort(). - * - * Implementations must reject an attempt to derive a key of size 0. - * - * \param[in,out] operation The key derivation operation object - * to set up. It must - * have been initialized but not set up yet. - * \param alg The key derivation algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_KEY_DERIVATION(\p alg) is true). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c alg is not a key derivation algorithm. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \c alg is not supported or is not a key derivation algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be inactive), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_setup( - psa_key_derivation_operation_t *operation, - psa_algorithm_t alg); - -/** Retrieve the current capacity of a key derivation operation. - * - * The capacity of a key derivation is the maximum number of bytes that it can - * return. When you get *N* bytes of output from a key derivation operation, - * this reduces its capacity by *N*. - * - * \param[in] operation The operation to query. - * \param[out] capacity On success, the capacity of the operation. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_get_capacity( - const psa_key_derivation_operation_t *operation, - size_t *capacity); - -/** Set the maximum capacity of a key derivation operation. - * - * The capacity of a key derivation operation is the maximum number of bytes - * that the key derivation operation can return from this point onwards. - * - * \param[in,out] operation The key derivation operation object to modify. - * \param capacity The new capacity of the operation. - * It must be less or equal to the operation's - * current capacity. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p capacity is larger than the operation's current capacity. - * In this case, the operation object remains valid and its capacity - * remains unchanged. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active), or the - * library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_set_capacity( - psa_key_derivation_operation_t *operation, - size_t capacity); - -/** Use the maximum possible capacity for a key derivation operation. - * - * Use this value as the capacity argument when setting up a key derivation - * to indicate that the operation should have the maximum possible capacity. - * The value of the maximum possible capacity depends on the key derivation - * algorithm. - */ -#define PSA_KEY_DERIVATION_UNLIMITED_CAPACITY ((size_t) (-1)) - -/** Provide an input for key derivation or key agreement. - * - * Which inputs are required and in what order depends on the algorithm. - * Refer to the documentation of each key derivation or key agreement - * algorithm for information. - * - * This function passes direct inputs, which is usually correct for - * non-secret inputs. To pass a secret input, which should be in a key - * object, call psa_key_derivation_input_key() instead of this function. - * Refer to the documentation of individual step types - * (`PSA_KEY_DERIVATION_INPUT_xxx` values of type ::psa_key_derivation_step_t) - * for more information. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to use. - * It must have been set up with - * psa_key_derivation_setup() and must not - * have produced any output yet. - * \param step Which step the input data is for. - * \param[in] data Input data to use. - * \param data_length Size of the \p data buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c step is not compatible with the operation's algorithm, or - * \c step does not allow direct inputs. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid for this input \p step, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_input_bytes( - psa_key_derivation_operation_t *operation, - psa_key_derivation_step_t step, - const uint8_t *data, - size_t data_length); - -/** Provide a numeric input for key derivation or key agreement. - * - * Which inputs are required and in what order depends on the algorithm. - * However, when an algorithm requires a particular order, numeric inputs - * usually come first as they tend to be configuration parameters. - * Refer to the documentation of each key derivation or key agreement - * algorithm for information. - * - * This function is used for inputs which are fixed-size non-negative - * integers. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to use. - * It must have been set up with - * psa_key_derivation_setup() and must not - * have produced any output yet. - * \param step Which step the input data is for. - * \param[in] value The value of the numeric input. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c step is not compatible with the operation's algorithm, or - * \c step does not allow numeric inputs. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid for this input \p step, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_input_integer( - psa_key_derivation_operation_t *operation, - psa_key_derivation_step_t step, - uint64_t value); - -/** Provide an input for key derivation in the form of a key. - * - * Which inputs are required and in what order depends on the algorithm. - * Refer to the documentation of each key derivation or key agreement - * algorithm for information. - * - * This function obtains input from a key object, which is usually correct for - * secret inputs or for non-secret personalization strings kept in the key - * store. To pass a non-secret parameter which is not in the key store, - * call psa_key_derivation_input_bytes() instead of this function. - * Refer to the documentation of individual step types - * (`PSA_KEY_DERIVATION_INPUT_xxx` values of type ::psa_key_derivation_step_t) - * for more information. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to use. - * It must have been set up with - * psa_key_derivation_setup() and must not - * have produced any output yet. - * \param step Which step the input data is for. - * \param key Identifier of the key. It must have an - * appropriate type for step and must allow the - * usage #PSA_KEY_USAGE_DERIVE or - * #PSA_KEY_USAGE_VERIFY_DERIVATION (see note) - * and the algorithm used by the operation. - * - * \note Once all inputs steps are completed, the operations will allow: - * - psa_key_derivation_output_bytes() if each input was either a direct input - * or a key with #PSA_KEY_USAGE_DERIVE set; - * - psa_key_derivation_output_key() or psa_key_derivation_output_key_custom() - * if the input for step - * #PSA_KEY_DERIVATION_INPUT_SECRET or #PSA_KEY_DERIVATION_INPUT_PASSWORD - * was from a key slot with #PSA_KEY_USAGE_DERIVE and each other input was - * either a direct input or a key with #PSA_KEY_USAGE_DERIVE set; - * - psa_key_derivation_verify_bytes() if each input was either a direct input - * or a key with #PSA_KEY_USAGE_VERIFY_DERIVATION set; - * - psa_key_derivation_verify_key() under the same conditions as - * psa_key_derivation_verify_bytes(). - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * The key allows neither #PSA_KEY_USAGE_DERIVE nor - * #PSA_KEY_USAGE_VERIFY_DERIVATION, or it doesn't allow this - * algorithm. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c step is not compatible with the operation's algorithm, or - * \c step does not allow key inputs of the given type - * or does not allow key inputs at all. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid for this input \p step, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_input_key( - psa_key_derivation_operation_t *operation, - psa_key_derivation_step_t step, - mbedtls_svc_key_id_t key); - -/** Perform a key agreement and use the shared secret as input to a key - * derivation. - * - * A key agreement algorithm takes two inputs: a private key \p private_key - * a public key \p peer_key. - * The result of this function is passed as input to a key derivation. - * The output of this key derivation can be extracted by reading from the - * resulting operation to produce keys and other cryptographic material. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to use. - * It must have been set up with - * psa_key_derivation_setup() with a - * key agreement and derivation algorithm - * \c alg (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_KEY_AGREEMENT(\c alg) is true - * and #PSA_ALG_IS_RAW_KEY_AGREEMENT(\c alg) - * is false). - * The operation must be ready for an - * input of the type given by \p step. - * \param step Which step the input data is for. - * \param private_key Identifier of the private key to use. It must - * allow the usage #PSA_KEY_USAGE_DERIVE. - * \param[in] peer_key Public key of the peer. The peer key must be in the - * same format that psa_import_key() accepts for the - * public key type corresponding to the type of - * private_key. That is, this function performs the - * equivalent of - * #psa_import_key(..., - * `peer_key`, `peer_key_length`) where - * with key attributes indicating the public key - * type corresponding to the type of `private_key`. - * For example, for EC keys, this means that peer_key - * is interpreted as a point on the curve that the - * private key is on. The standard formats for public - * keys are documented in the documentation of - * psa_export_public_key(). - * \param peer_key_length Size of \p peer_key in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \c private_key is not compatible with \c alg, - * or \p peer_key is not valid for \c alg or not compatible with - * \c private_key, or \c step does not allow an input resulting - * from a key agreement. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \c alg is not supported or is not a key derivation algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid for this key agreement \p step, - * or the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_key_agreement( - psa_key_derivation_operation_t *operation, - psa_key_derivation_step_t step, - mbedtls_svc_key_id_t private_key, - const uint8_t *peer_key, - size_t peer_key_length); - -/** Read some data from a key derivation operation. - * - * This function calculates output bytes from a key derivation algorithm and - * return those bytes. - * If you view the key derivation's output as a stream of bytes, this - * function destructively reads the requested number of bytes from the - * stream. - * The operation's capacity decreases by the number of bytes read. - * - * If this function returns an error status other than - * #PSA_ERROR_INSUFFICIENT_DATA, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to read from. - * \param[out] output Buffer where the output will be written. - * \param output_length Number of bytes to output. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * One of the inputs was a key whose policy didn't allow - * #PSA_KEY_USAGE_DERIVE. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * The operation's capacity was less than - * \p output_length bytes. Note that in this case, - * no output is written to the output buffer. - * The operation's capacity is set to 0, thus - * subsequent calls to this function will not - * succeed, even with a smaller output buffer. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_output_bytes( - psa_key_derivation_operation_t *operation, - uint8_t *output, - size_t output_length); - -/** Derive a key from an ongoing key derivation operation. - * - * This function calculates output bytes from a key derivation algorithm - * and uses those bytes to generate a key deterministically. - * The key's location, usage policy, type and size are taken from - * \p attributes. - * - * If you view the key derivation's output as a stream of bytes, this - * function destructively reads as many bytes as required from the - * stream. - * The operation's capacity decreases by the number of bytes read. - * - * If this function returns an error status other than - * #PSA_ERROR_INSUFFICIENT_DATA, the operation enters an error - * state and must be aborted by calling psa_key_derivation_abort(). - * - * How much output is produced and consumed from the operation, and how - * the key is derived, depends on the key type and on the key size - * (denoted \c bits below): - * - * - For key types for which the key is an arbitrary sequence of bytes - * of a given size, this function is functionally equivalent to - * calling #psa_key_derivation_output_bytes - * and passing the resulting output to #psa_import_key. - * However, this function has a security benefit: - * if the implementation provides an isolation boundary then - * the key material is not exposed outside the isolation boundary. - * As a consequence, for these key types, this function always consumes - * exactly (\c bits / 8) bytes from the operation. - * The following key types defined in this specification follow this scheme: - * - * - #PSA_KEY_TYPE_AES; - * - #PSA_KEY_TYPE_ARIA; - * - #PSA_KEY_TYPE_CAMELLIA; - * - #PSA_KEY_TYPE_DERIVE; - * - #PSA_KEY_TYPE_HMAC; - * - #PSA_KEY_TYPE_PASSWORD_HASH. - * - * - For ECC keys on a Montgomery elliptic curve - * (#PSA_KEY_TYPE_ECC_KEY_PAIR(\c curve) where \c curve designates a - * Montgomery curve), this function always draws a byte string whose - * length is determined by the curve, and sets the mandatory bits - * accordingly. That is: - * - * - Curve25519 (#PSA_ECC_FAMILY_MONTGOMERY, 255 bits): draw a 32-byte - * string and process it as specified in RFC 7748 §5. - * - Curve448 (#PSA_ECC_FAMILY_MONTGOMERY, 448 bits): draw a 56-byte - * string and process it as specified in RFC 7748 §5. - * - * - For key types for which the key is represented by a single sequence of - * \c bits bits with constraints as to which bit sequences are acceptable, - * this function draws a byte string of length (\c bits / 8) bytes rounded - * up to the nearest whole number of bytes. If the resulting byte string - * is acceptable, it becomes the key, otherwise the drawn bytes are discarded. - * This process is repeated until an acceptable byte string is drawn. - * The byte string drawn from the operation is interpreted as specified - * for the output produced by psa_export_key(). - * The following key types defined in this specification follow this scheme: - * - * - #PSA_KEY_TYPE_DES. - * Force-set the parity bits, but discard forbidden weak keys. - * For 2-key and 3-key triple-DES, the three keys are generated - * successively (for example, for 3-key triple-DES, - * if the first 8 bytes specify a weak key and the next 8 bytes do not, - * discard the first 8 bytes, use the next 8 bytes as the first key, - * and continue reading output from the operation to derive the other - * two keys). - * - Finite-field Diffie-Hellman keys (#PSA_KEY_TYPE_DH_KEY_PAIR(\c group) - * where \c group designates any Diffie-Hellman group) and - * ECC keys on a Weierstrass elliptic curve - * (#PSA_KEY_TYPE_ECC_KEY_PAIR(\c curve) where \c curve designates a - * Weierstrass curve). - * For these key types, interpret the byte string as integer - * in big-endian order. Discard it if it is not in the range - * [0, *N* - 2] where *N* is the boundary of the private key domain - * (the prime *p* for Diffie-Hellman, the subprime *q* for DSA, - * or the order of the curve's base point for ECC). - * Add 1 to the resulting integer and use this as the private key *x*. - * This method allows compliance to NIST standards, specifically - * the methods titled "key-pair generation by testing candidates" - * in NIST SP 800-56A §5.6.1.1.4 for Diffie-Hellman, - * in FIPS 186-4 §B.1.2 for DSA, and - * in NIST SP 800-56A §5.6.1.2.2 or - * FIPS 186-4 §B.4.2 for elliptic curve keys. - * - * - For other key types, including #PSA_KEY_TYPE_RSA_KEY_PAIR, - * the way in which the operation output is consumed is - * implementation-defined. - * - * In all cases, the data that is read is discarded from the operation. - * The operation's capacity is decreased by the number of bytes read. - * - * For algorithms that take an input step #PSA_KEY_DERIVATION_INPUT_SECRET, - * the input to that step must be provided with psa_key_derivation_input_key(). - * Future versions of this specification may include additional restrictions - * on the derived key based on the attributes and strength of the secret key. - * - * \note This function is equivalent to calling - * psa_key_derivation_output_key_custom() - * with the custom production parameters #PSA_CUSTOM_KEY_PARAMETERS_INIT - * and `custom_data_length == 0` (i.e. `custom_data` is empty). - * - * \param[in] attributes The attributes for the new key. - * If the key type to be created is - * #PSA_KEY_TYPE_PASSWORD_HASH then the algorithm in - * the policy must be the same as in the current - * operation. - * \param[in,out] operation The key derivation operation object to read from. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * There was not enough data to create the desired key. - * Note that in this case, no output is written to the output buffer. - * The operation's capacity is set to 0, thus subsequent calls to - * this function will not succeed, even with a smaller output buffer. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The key type or key size is not supported, either by the - * implementation in general or in this particular location. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The provided key attributes are not valid for the operation. - * \retval #PSA_ERROR_NOT_PERMITTED - * The #PSA_KEY_DERIVATION_INPUT_SECRET or - * #PSA_KEY_DERIVATION_INPUT_PASSWORD input was not provided through a - * key; or one of the inputs was a key whose policy didn't allow - * #PSA_KEY_USAGE_DERIVE. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_output_key( - const psa_key_attributes_t *attributes, - psa_key_derivation_operation_t *operation, - mbedtls_svc_key_id_t *key); - -/** Derive a key from an ongoing key derivation operation with custom - * production parameters. - * - * See the description of psa_key_derivation_out_key() for the operation of - * this function with the default production parameters. - * Mbed TLS currently does not currently support any non-default production - * parameters. - * - * \note This function is experimental and may change in future minor - * versions of Mbed TLS. - * - * \param[in] attributes The attributes for the new key. - * If the key type to be created is - * #PSA_KEY_TYPE_PASSWORD_HASH then the algorithm in - * the policy must be the same as in the current - * operation. - * \param[in,out] operation The key derivation operation object to read from. - * \param[in] custom Customization parameters for the key generation. - * When this is #PSA_CUSTOM_KEY_PARAMETERS_INIT - * with \p custom_data_length = 0, - * this function is equivalent to - * psa_key_derivation_output_key(). - * \param[in] custom_data Variable-length data associated with \c custom. - * \param custom_data_length - * Length of `custom_data` in bytes. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * There was not enough data to create the desired key. - * Note that in this case, no output is written to the output buffer. - * The operation's capacity is set to 0, thus subsequent calls to - * this function will not succeed, even with a smaller output buffer. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The key type or key size is not supported, either by the - * implementation in general or in this particular location. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The provided key attributes are not valid for the operation. - * \retval #PSA_ERROR_NOT_PERMITTED - * The #PSA_KEY_DERIVATION_INPUT_SECRET or - * #PSA_KEY_DERIVATION_INPUT_PASSWORD input was not provided through a - * key; or one of the inputs was a key whose policy didn't allow - * #PSA_KEY_USAGE_DERIVE. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_output_key_custom( - const psa_key_attributes_t *attributes, - psa_key_derivation_operation_t *operation, - const psa_custom_key_parameters_t *custom, - const uint8_t *custom_data, - size_t custom_data_length, - mbedtls_svc_key_id_t *key); - -#ifndef __cplusplus -/* Omitted when compiling in C++, because one of the parameters is a - * pointer to a struct with a flexible array member, and that is not - * standard C++. - * https://github.com/Mbed-TLS/mbedtls/issues/9020 - */ -/** Derive a key from an ongoing key derivation operation with custom - * production parameters. - * - * \note - * This is a deprecated variant of psa_key_derivation_output_key_custom(). - * It is equivalent except that the associated variable-length data - * is passed in `params->data` instead of a separate parameter. - * This function will be removed in a future version of Mbed TLS. - * - * \param[in] attributes The attributes for the new key. - * If the key type to be created is - * #PSA_KEY_TYPE_PASSWORD_HASH then the algorithm in - * the policy must be the same as in the current - * operation. - * \param[in,out] operation The key derivation operation object to read from. - * \param[in] params Customization parameters for the key derivation. - * When this is #PSA_KEY_PRODUCTION_PARAMETERS_INIT - * with \p params_data_length = 0, - * this function is equivalent to - * psa_key_derivation_output_key(). - * Mbed TLS currently only supports the default - * production parameters, i.e. - * #PSA_KEY_PRODUCTION_PARAMETERS_INIT, - * for all key types. - * \param params_data_length - * Length of `params->data` in bytes. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * There was not enough data to create the desired key. - * Note that in this case, no output is written to the output buffer. - * The operation's capacity is set to 0, thus subsequent calls to - * this function will not succeed, even with a smaller output buffer. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The key type or key size is not supported, either by the - * implementation in general or in this particular location. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The provided key attributes are not valid for the operation. - * \retval #PSA_ERROR_NOT_PERMITTED - * The #PSA_KEY_DERIVATION_INPUT_SECRET or - * #PSA_KEY_DERIVATION_INPUT_PASSWORD input was not provided through a - * key; or one of the inputs was a key whose policy didn't allow - * #PSA_KEY_USAGE_DERIVE. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_output_key_ext( - const psa_key_attributes_t *attributes, - psa_key_derivation_operation_t *operation, - const psa_key_production_parameters_t *params, - size_t params_data_length, - mbedtls_svc_key_id_t *key); -#endif /* !__cplusplus */ - -/** Compare output data from a key derivation operation to an expected value. - * - * This function calculates output bytes from a key derivation algorithm and - * compares those bytes to an expected value in constant time. - * If you view the key derivation's output as a stream of bytes, this - * function destructively reads the expected number of bytes from the - * stream before comparing them. - * The operation's capacity decreases by the number of bytes read. - * - * This is functionally equivalent to the following code: - * \code - * psa_key_derivation_output_bytes(operation, tmp, output_length); - * if (memcmp(output, tmp, output_length) != 0) - * return PSA_ERROR_INVALID_SIGNATURE; - * \endcode - * except (1) it works even if the key's policy does not allow outputting the - * bytes, and (2) the comparison will be done in constant time. - * - * If this function returns an error status other than - * #PSA_ERROR_INSUFFICIENT_DATA or #PSA_ERROR_INVALID_SIGNATURE, - * the operation enters an error state and must be aborted by calling - * psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to read from. - * \param[in] expected Buffer containing the expected derivation output. - * \param expected_length Length of the expected output; this is also the - * number of bytes that will be read. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The output was read successfully, but it differs from the expected - * output. - * \retval #PSA_ERROR_NOT_PERMITTED - * One of the inputs was a key whose policy didn't allow - * #PSA_KEY_USAGE_VERIFY_DERIVATION. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * The operation's capacity was less than - * \p output_length bytes. Note that in this case, - * the operation's capacity is set to 0, thus - * subsequent calls to this function will not - * succeed, even with a smaller expected output. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_verify_bytes( - psa_key_derivation_operation_t *operation, - const uint8_t *expected, - size_t expected_length); - -/** Compare output data from a key derivation operation to an expected value - * stored in a key object. - * - * This function calculates output bytes from a key derivation algorithm and - * compares those bytes to an expected value, provided as key of type - * #PSA_KEY_TYPE_PASSWORD_HASH. - * If you view the key derivation's output as a stream of bytes, this - * function destructively reads the number of bytes corresponding to the - * length of the expected value from the stream before comparing them. - * The operation's capacity decreases by the number of bytes read. - * - * This is functionally equivalent to exporting the key and calling - * psa_key_derivation_verify_bytes() on the result, except that it - * works even if the key cannot be exported. - * - * If this function returns an error status other than - * #PSA_ERROR_INSUFFICIENT_DATA or #PSA_ERROR_INVALID_SIGNATURE, - * the operation enters an error state and must be aborted by calling - * psa_key_derivation_abort(). - * - * \param[in,out] operation The key derivation operation object to read from. - * \param[in] expected A key of type #PSA_KEY_TYPE_PASSWORD_HASH - * containing the expected output. Its policy must - * include the #PSA_KEY_USAGE_VERIFY_DERIVATION flag - * and the permitted algorithm must match the - * operation. The value of this key was likely - * computed by a previous call to - * psa_key_derivation_output_key() or - * psa_key_derivation_output_key_custom(). - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The output was read successfully, but if differs from the expected - * output. - * \retval #PSA_ERROR_INVALID_HANDLE - * The key passed as the expected value does not exist. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The key passed as the expected value has an invalid type. - * \retval #PSA_ERROR_NOT_PERMITTED - * The key passed as the expected value does not allow this usage or - * this algorithm; or one of the inputs was a key whose policy didn't - * allow #PSA_KEY_USAGE_VERIFY_DERIVATION. - * \retval #PSA_ERROR_INSUFFICIENT_DATA - * The operation's capacity was less than - * the length of the expected value. In this case, - * the operation's capacity is set to 0, thus - * subsequent calls to this function will not - * succeed, even with a smaller expected output. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active and completed - * all required input steps), or the library has not been previously - * initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_verify_key( - psa_key_derivation_operation_t *operation, - psa_key_id_t expected); - -/** Abort a key derivation operation. - * - * Aborting an operation frees all associated resources except for the \c - * operation structure itself. Once aborted, the operation object can be reused - * for another operation by calling psa_key_derivation_setup() again. - * - * This function may be called at any time after the operation - * object has been initialized as described in #psa_key_derivation_operation_t. - * - * In particular, it is valid to call psa_key_derivation_abort() twice, or to - * call psa_key_derivation_abort() on an operation that has not been set up. - * - * \param[in,out] operation The operation to abort. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_key_derivation_abort( - psa_key_derivation_operation_t *operation); - -/** Perform a key agreement and return the raw shared secret. - * - * \warning The raw result of a key agreement algorithm such as finite-field - * Diffie-Hellman or elliptic curve Diffie-Hellman has biases and should - * not be used directly as key material. It should instead be passed as - * input to a key derivation algorithm. To chain a key agreement with - * a key derivation, use psa_key_derivation_key_agreement() and other - * functions from the key derivation interface. - * - * \param alg The key agreement algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_RAW_KEY_AGREEMENT(\p alg) - * is true). - * \param private_key Identifier of the private key to use. It must - * allow the usage #PSA_KEY_USAGE_DERIVE. - * \param[in] peer_key Public key of the peer. It must be - * in the same format that psa_import_key() - * accepts. The standard formats for public - * keys are documented in the documentation - * of psa_export_public_key(). - * \param peer_key_length Size of \p peer_key in bytes. - * \param[out] output Buffer where the decrypted message is to - * be written. - * \param output_size Size of the \c output buffer in bytes. - * \param[out] output_length On success, the number of bytes - * that make up the returned output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p alg is not a key agreement algorithm, or - * \p private_key is not compatible with \p alg, - * or \p peer_key is not valid for \p alg or not compatible with - * \p private_key. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * \p output_size is too small - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p alg is not a supported key agreement algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_raw_key_agreement(psa_algorithm_t alg, - mbedtls_svc_key_id_t private_key, - const uint8_t *peer_key, - size_t peer_key_length, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/**@}*/ - -/** \defgroup random Random generation - * @{ - */ - -/** - * \brief Generate random bytes. - * - * \warning This function **can** fail! Callers MUST check the return status - * and MUST NOT use the content of the output buffer if the return - * status is not #PSA_SUCCESS. - * - * \note To generate a key, use psa_generate_key() instead. - * - * \param[out] output Output buffer for the generated data. - * \param output_size Number of bytes to generate and output. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_generate_random(uint8_t *output, - size_t output_size); - -/** - * \brief Generate a key or key pair. - * - * The key is generated randomly. - * Its location, usage policy, type and size are taken from \p attributes. - * - * Implementations must reject an attempt to generate a key of size 0. - * - * The following type-specific considerations apply: - * - For RSA keys (#PSA_KEY_TYPE_RSA_KEY_PAIR), - * the public exponent is 65537. - * The modulus is a product of two probabilistic primes - * between 2^{n-1} and 2^n where n is the bit size specified in the - * attributes. - * - * \note This function is equivalent to calling psa_generate_key_custom() - * with the custom production parameters #PSA_CUSTOM_KEY_PARAMETERS_INIT - * and `custom_data_length == 0` (i.e. `custom_data` is empty). - * - * \param[in] attributes The attributes for the new key. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_generate_key(const psa_key_attributes_t *attributes, - mbedtls_svc_key_id_t *key); - -/** - * \brief Generate a key or key pair using custom production parameters. - * - * See the description of psa_generate_key() for the operation of this - * function with the default production parameters. In addition, this function - * supports the following production customizations, described in more detail - * in the documentation of ::psa_custom_key_parameters_t: - * - * - RSA keys: generation with a custom public exponent. - * - * \note This function is experimental and may change in future minor - * versions of Mbed TLS. - * - * \param[in] attributes The attributes for the new key. - * \param[in] custom Customization parameters for the key generation. - * When this is #PSA_CUSTOM_KEY_PARAMETERS_INIT - * with \p custom_data_length = 0, - * this function is equivalent to - * psa_generate_key(). - * \param[in] custom_data Variable-length data associated with \c custom. - * \param custom_data_length - * Length of `custom_data` in bytes. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_generate_key_custom(const psa_key_attributes_t *attributes, - const psa_custom_key_parameters_t *custom, - const uint8_t *custom_data, - size_t custom_data_length, - mbedtls_svc_key_id_t *key); - -#ifndef __cplusplus -/* Omitted when compiling in C++, because one of the parameters is a - * pointer to a struct with a flexible array member, and that is not - * standard C++. - * https://github.com/Mbed-TLS/mbedtls/issues/9020 - */ -/** - * \brief Generate a key or key pair using custom production parameters. - * - * \note - * This is a deprecated variant of psa_key_derivation_output_key_custom(). - * It is equivalent except that the associated variable-length data - * is passed in `params->data` instead of a separate parameter. - * This function will be removed in a future version of Mbed TLS. - * - * \param[in] attributes The attributes for the new key. - * \param[in] params Customization parameters for the key generation. - * When this is #PSA_KEY_PRODUCTION_PARAMETERS_INIT - * with \p params_data_length = 0, - * this function is equivalent to - * psa_generate_key(). - * \param params_data_length - * Length of `params->data` in bytes. - * \param[out] key On success, an identifier for the newly created - * key. For persistent keys, this is the key - * identifier defined in \p attributes. - * \c 0 on failure. - * - * \retval #PSA_SUCCESS - * Success. - * If the key is persistent, the key material and the key's metadata - * have been saved to persistent storage. - * \retval #PSA_ERROR_ALREADY_EXISTS - * This is an attempt to create a persistent key, and there is - * already a persistent key with the given identifier. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_generate_key_ext(const psa_key_attributes_t *attributes, - const psa_key_production_parameters_t *params, - size_t params_data_length, - mbedtls_svc_key_id_t *key); -#endif /* !__cplusplus */ - -/**@}*/ - -/** \defgroup interruptible_hash Interruptible sign/verify hash - * @{ - */ - -/** The type of the state data structure for interruptible hash - * signing operations. - * - * Before calling any function on a sign hash operation object, the - * application must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_sign_hash_interruptible_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_sign_hash_interruptible_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer - * #PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT, for example: - * \code - * psa_sign_hash_interruptible_operation_t operation = - * PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT; - * \endcode - * - Assign the result of the function - * psa_sign_hash_interruptible_operation_init() to the structure, for - * example: - * \code - * psa_sign_hash_interruptible_operation_t operation; - * operation = psa_sign_hash_interruptible_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_sign_hash_interruptible_operation_s psa_sign_hash_interruptible_operation_t; - -/** The type of the state data structure for interruptible hash - * verification operations. - * - * Before calling any function on a sign hash operation object, the - * application must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_verify_hash_interruptible_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_verify_hash_interruptible_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer - * #PSA_VERIFY_HASH_INTERRUPTIBLE_OPERATION_INIT, for example: - * \code - * psa_verify_hash_interruptible_operation_t operation = - * PSA_VERIFY_HASH_INTERRUPTIBLE_OPERATION_INIT; - * \endcode - * - Assign the result of the function - * psa_verify_hash_interruptible_operation_init() to the structure, for - * example: - * \code - * psa_verify_hash_interruptible_operation_t operation; - * operation = psa_verify_hash_interruptible_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_verify_hash_interruptible_operation_s psa_verify_hash_interruptible_operation_t; - -/** - * \brief Set the maximum number of ops allowed to be - * executed by an interruptible function in a - * single call. - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note The time taken to execute a single op is - * implementation specific and depends on - * software, hardware, the algorithm, key type and - * curve chosen. Even within a single operation, - * successive ops can take differing amounts of - * time. The only guarantee is that lower values - * for \p max_ops means functions will block for a - * lesser maximum amount of time. The functions - * \c psa_sign_interruptible_get_num_ops() and - * \c psa_verify_interruptible_get_num_ops() are - * provided to help with tuning this value. - * - * \note This value defaults to - * #PSA_INTERRUPTIBLE_MAX_OPS_UNLIMITED, which - * means the whole operation will be done in one - * go, regardless of the number of ops required. - * - * \note If more ops are needed to complete a - * computation, #PSA_OPERATION_INCOMPLETE will be - * returned by the function performing the - * computation. It is then the caller's - * responsibility to either call again with the - * same operation context until it returns 0 or an - * error code; or to call the relevant abort - * function if the answer is no longer required. - * - * \note The interpretation of \p max_ops is also - * implementation defined. On a hard real time - * system, this can indicate a hard deadline, as a - * real-time system needs a guarantee of not - * spending more than X time, however care must be - * taken in such an implementation to avoid the - * situation whereby calls just return, not being - * able to do any actual work within the allotted - * time. On a non-real-time system, the - * implementation can be more relaxed, but again - * whether this number should be interpreted as as - * hard or soft limit or even whether a less than - * or equals as regards to ops executed in a - * single call is implementation defined. - * - * \note For keys in local storage when no accelerator - * driver applies, please see also the - * documentation for \c mbedtls_ecp_set_max_ops(), - * which is the internal implementation in these - * cases. - * - * \warning With implementations that interpret this number - * as a hard limit, setting this number too small - * may result in an infinite loop, whereby each - * call results in immediate return with no ops - * done (as there is not enough time to execute - * any), and thus no result will ever be achieved. - * - * \note This only applies to functions whose - * documentation mentions they may return - * #PSA_OPERATION_INCOMPLETE. - * - * \param max_ops The maximum number of ops to be executed in a - * single call. This can be a number from 0 to - * #PSA_INTERRUPTIBLE_MAX_OPS_UNLIMITED, where 0 - * is the least amount of work done per call. - */ -void psa_interruptible_set_max_ops(uint32_t max_ops); - -/** - * \brief Get the maximum number of ops allowed to be - * executed by an interruptible function in a - * single call. This will return the last - * value set by - * \c psa_interruptible_set_max_ops() or - * #PSA_INTERRUPTIBLE_MAX_OPS_UNLIMITED if - * that function has never been called. - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \return Maximum number of ops allowed to be - * executed by an interruptible function in a - * single call. - */ -uint32_t psa_interruptible_get_max_ops(void); - -/** - * \brief Get the number of ops that a hash signing - * operation has taken so far. If the operation - * has completed, then this will represent the - * number of ops required for the entire - * operation. After initialization or calling - * \c psa_sign_hash_interruptible_abort() on - * the operation, a value of 0 will be returned. - * - * \note This interface is guaranteed re-entrant and - * thus may be called from driver code. - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * This is a helper provided to help you tune the - * value passed to \c - * psa_interruptible_set_max_ops(). - * - * \param operation The \c psa_sign_hash_interruptible_operation_t - * to use. This must be initialized first. - * - * \return Number of ops that the operation has taken so - * far. - */ -uint32_t psa_sign_hash_get_num_ops( - const psa_sign_hash_interruptible_operation_t *operation); - -/** - * \brief Get the number of ops that a hash verification - * operation has taken so far. If the operation - * has completed, then this will represent the - * number of ops required for the entire - * operation. After initialization or calling \c - * psa_verify_hash_interruptible_abort() on the - * operation, a value of 0 will be returned. - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * This is a helper provided to help you tune the - * value passed to \c - * psa_interruptible_set_max_ops(). - * - * \param operation The \c - * psa_verify_hash_interruptible_operation_t to - * use. This must be initialized first. - * - * \return Number of ops that the operation has taken so - * far. - */ -uint32_t psa_verify_hash_get_num_ops( - const psa_verify_hash_interruptible_operation_t *operation); - -/** - * \brief Start signing a hash or short message with a - * private key, in an interruptible manner. - * - * \see \c psa_sign_hash_complete() - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note This function combined with \c - * psa_sign_hash_complete() is equivalent to - * \c psa_sign_hash() but - * \c psa_sign_hash_complete() can return early and - * resume according to the limit set with \c - * psa_interruptible_set_max_ops() to reduce the - * maximum time spent in a function call. - * - * \note Users should call \c psa_sign_hash_complete() - * repeatedly on the same context after a - * successful call to this function until \c - * psa_sign_hash_complete() either returns 0 or an - * error. \c psa_sign_hash_complete() will return - * #PSA_OPERATION_INCOMPLETE if there is more work - * to do. Alternatively users can call - * \c psa_sign_hash_abort() at any point if they no - * longer want the result. - * - * \note If this function returns an error status, the - * operation enters an error state and must be - * aborted by calling \c psa_sign_hash_abort(). - * - * \param[in, out] operation The \c psa_sign_hash_interruptible_operation_t - * to use. This must be initialized first. - * - * \param key Identifier of the key to use for the operation. - * It must be an asymmetric key pair. The key must - * allow the usage #PSA_KEY_USAGE_SIGN_HASH. - * \param alg A signature algorithm (\c PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_HASH(\p alg) - * is true), that is compatible with - * the type of \p key. - * \param[in] hash The hash or message to sign. - * \param hash_length Size of the \p hash buffer in bytes. - * - * \retval #PSA_SUCCESS - * The operation started successfully - call \c psa_sign_hash_complete() - * with the same context to complete the operation - * - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_SIGN_HASH flag, or it does - * not permit the requested algorithm. - * \retval #PSA_ERROR_BAD_STATE - * An operation has previously been started on this context, and is - * still in progress. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_sign_hash_start( - psa_sign_hash_interruptible_operation_t *operation, - mbedtls_svc_key_id_t key, psa_algorithm_t alg, - const uint8_t *hash, size_t hash_length); - -/** - * \brief Continue and eventually complete the action of - * signing a hash or short message with a private - * key, in an interruptible manner. - * - * \see \c psa_sign_hash_start() - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note This function combined with \c - * psa_sign_hash_start() is equivalent to - * \c psa_sign_hash() but this function can return - * early and resume according to the limit set with - * \c psa_interruptible_set_max_ops() to reduce the - * maximum time spent in a function call. - * - * \note Users should call this function on the same - * operation object repeatedly until it either - * returns 0 or an error. This function will return - * #PSA_OPERATION_INCOMPLETE if there is more work - * to do. Alternatively users can call - * \c psa_sign_hash_abort() at any point if they no - * longer want the result. - * - * \note When this function returns successfully, the - * operation becomes inactive. If this function - * returns an error status, the operation enters an - * error state and must be aborted by calling - * \c psa_sign_hash_abort(). - * - * \param[in, out] operation The \c psa_sign_hash_interruptible_operation_t - * to use. This must be initialized first, and have - * had \c psa_sign_hash_start() called with it - * first. - * - * \param[out] signature Buffer where the signature is to be written. - * \param signature_size Size of the \p signature buffer in bytes. This - * must be appropriate for the selected - * algorithm and key: - * - The required signature size is - * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c - * key_bits, \c alg) where \c key_type and \c - * key_bits are the type and bit-size - * respectively of key. - * - #PSA_SIGNATURE_MAX_SIZE evaluates to the - * maximum signature size of any supported - * signature algorithm. - * \param[out] signature_length On success, the number of bytes that make up - * the returned signature value. - * - * \retval #PSA_SUCCESS - * Operation completed successfully - * - * \retval #PSA_OPERATION_INCOMPLETE - * Operation was interrupted due to the setting of \c - * psa_interruptible_set_max_ops(). There is still work to be done. - * Call this function again with the same operation object. - * - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p signature buffer is too small. You can - * determine a sufficient buffer size by calling - * #PSA_SIGN_OUTPUT_SIZE(\c key_type, \c key_bits, \c alg) - * where \c key_type and \c key_bits are the type and bit-size - * respectively of \c key. - * - * \retval #PSA_ERROR_BAD_STATE - * An operation was not previously started on this context via - * \c psa_sign_hash_start(). - * - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has either not been previously initialized by - * psa_crypto_init() or you did not previously call - * psa_sign_hash_start() with this operation object. It is - * implementation-dependent whether a failure to initialize results in - * this error code. - */ -psa_status_t psa_sign_hash_complete( - psa_sign_hash_interruptible_operation_t *operation, - uint8_t *signature, size_t signature_size, - size_t *signature_length); - -/** - * \brief Abort a sign hash operation. - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note This function is the only function that clears - * the number of ops completed as part of the - * operation. Please ensure you copy this value via - * \c psa_sign_hash_get_num_ops() if required - * before calling. - * - * \note Aborting an operation frees all associated - * resources except for the \p operation structure - * itself. Once aborted, the operation object can - * be reused for another operation by calling \c - * psa_sign_hash_start() again. - * - * \note You may call this function any time after the - * operation object has been initialized. In - * particular, calling \c psa_sign_hash_abort() - * after the operation has already been terminated - * by a call to \c psa_sign_hash_abort() or - * psa_sign_hash_complete() is safe. - * - * \param[in,out] operation Initialized sign hash operation. - * - * \retval #PSA_SUCCESS - * The operation was aborted successfully. - * - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_sign_hash_abort( - psa_sign_hash_interruptible_operation_t *operation); - -/** - * \brief Start reading and verifying a hash or short - * message, in an interruptible manner. - * - * \see \c psa_verify_hash_complete() - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note This function combined with \c - * psa_verify_hash_complete() is equivalent to - * \c psa_verify_hash() but \c - * psa_verify_hash_complete() can return early and - * resume according to the limit set with \c - * psa_interruptible_set_max_ops() to reduce the - * maximum time spent in a function. - * - * \note Users should call \c psa_verify_hash_complete() - * repeatedly on the same operation object after a - * successful call to this function until \c - * psa_verify_hash_complete() either returns 0 or - * an error. \c psa_verify_hash_complete() will - * return #PSA_OPERATION_INCOMPLETE if there is - * more work to do. Alternatively users can call - * \c psa_verify_hash_abort() at any point if they - * no longer want the result. - * - * \note If this function returns an error status, the - * operation enters an error state and must be - * aborted by calling \c psa_verify_hash_abort(). - * - * \param[in, out] operation The \c psa_verify_hash_interruptible_operation_t - * to use. This must be initialized first. - * - * \param key Identifier of the key to use for the operation. - * The key must allow the usage - * #PSA_KEY_USAGE_VERIFY_HASH. - * \param alg A signature algorithm (\c PSA_ALG_XXX - * value such that #PSA_ALG_IS_SIGN_HASH(\p alg) - * is true), that is compatible with - * the type of \p key. - * \param[in] hash The hash whose signature is to be verified. - * \param hash_length Size of the \p hash buffer in bytes. - * \param[in] signature Buffer containing the signature to verify. - * \param signature_length Size of the \p signature buffer in bytes. - * - * \retval #PSA_SUCCESS - * The operation started successfully - please call \c - * psa_verify_hash_complete() with the same context to complete the - * operation. - * - * \retval #PSA_ERROR_BAD_STATE - * Another operation has already been started on this context, and is - * still in progress. - * - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_VERIFY_HASH flag, or it does - * not permit the requested algorithm. - * - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_verify_hash_start( - psa_verify_hash_interruptible_operation_t *operation, - mbedtls_svc_key_id_t key, psa_algorithm_t alg, - const uint8_t *hash, size_t hash_length, - const uint8_t *signature, size_t signature_length); - -/** - * \brief Continue and eventually complete the action of - * reading and verifying a hash or short message - * signed with a private key, in an interruptible - * manner. - * - * \see \c psa_verify_hash_start() - * - * \warning This is a beta API, and thus subject to change - * at any point. It is not bound by the usual - * interface stability promises. - * - * \note This function combined with \c - * psa_verify_hash_start() is equivalent to - * \c psa_verify_hash() but this function can - * return early and resume according to the limit - * set with \c psa_interruptible_set_max_ops() to - * reduce the maximum time spent in a function - * call. - * - * \note Users should call this function on the same - * operation object repeatedly until it either - * returns 0 or an error. This function will return - * #PSA_OPERATION_INCOMPLETE if there is more work - * to do. Alternatively users can call - * \c psa_verify_hash_abort() at any point if they - * no longer want the result. - * - * \note When this function returns successfully, the - * operation becomes inactive. If this function - * returns an error status, the operation enters an - * error state and must be aborted by calling - * \c psa_verify_hash_abort(). - * - * \param[in, out] operation The \c psa_verify_hash_interruptible_operation_t - * to use. This must be initialized first, and have - * had \c psa_verify_hash_start() called with it - * first. - * - * \retval #PSA_SUCCESS - * Operation completed successfully, and the passed signature is valid. - * - * \retval #PSA_OPERATION_INCOMPLETE - * Operation was interrupted due to the setting of \c - * psa_interruptible_set_max_ops(). There is still work to be done. - * Call this function again with the same operation object. - * - * \retval #PSA_ERROR_INVALID_HANDLE \emptydescription - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The calculation was performed successfully, but the passed - * signature is not a valid signature. - * \retval #PSA_ERROR_BAD_STATE - * An operation was not previously started on this context via - * \c psa_verify_hash_start(). - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INVALID_ARGUMENT \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has either not been previously initialized by - * psa_crypto_init() or you did not previously call - * psa_verify_hash_start() on this object. It is - * implementation-dependent whether a failure to initialize results in - * this error code. - */ -psa_status_t psa_verify_hash_complete( - psa_verify_hash_interruptible_operation_t *operation); - -/** - * \brief Abort a verify hash operation. - * - * \warning This is a beta API, and thus subject to change at - * any point. It is not bound by the usual interface - * stability promises. - * - * \note This function is the only function that clears the - * number of ops completed as part of the operation. - * Please ensure you copy this value via - * \c psa_verify_hash_get_num_ops() if required - * before calling. - * - * \note Aborting an operation frees all associated - * resources except for the operation structure - * itself. Once aborted, the operation object can be - * reused for another operation by calling \c - * psa_verify_hash_start() again. - * - * \note You may call this function any time after the - * operation object has been initialized. - * In particular, calling \c psa_verify_hash_abort() - * after the operation has already been terminated by - * a call to \c psa_verify_hash_abort() or - * psa_verify_hash_complete() is safe. - * - * \param[in,out] operation Initialized verify hash operation. - * - * \retval #PSA_SUCCESS - * The operation was aborted successfully. - * - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_verify_hash_abort( - psa_verify_hash_interruptible_operation_t *operation); - - -/**@}*/ - -#ifdef __cplusplus -} -#endif - -/* The file "crypto_extra.h" contains vendor-specific definitions. This - * can include vendor-defined algorithms, extra functions, etc. */ -#include "crypto_extra.h" - -#endif /* PSA_CRYPTO_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_auto_enabled.h b/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_auto_enabled.h deleted file mode 100644 index 3a2af1518..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_auto_enabled.h +++ /dev/null @@ -1,31 +0,0 @@ -/** - * \file psa/crypto_adjust_auto_enabled.h - * \brief Adjust PSA configuration: enable always-on features - * - * This is an internal header. Do not include it directly. - * - * Always enable certain features which require a negligible amount of code - * to implement, to avoid some edge cases in the configuration combinatorics. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_ADJUST_AUTO_ENABLED_H -#define PSA_CRYPTO_ADJUST_AUTO_ENABLED_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include psa/crypto_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -#define PSA_WANT_KEY_TYPE_DERIVE 1 -#define PSA_WANT_KEY_TYPE_PASSWORD 1 -#define PSA_WANT_KEY_TYPE_PASSWORD_HASH 1 -#define PSA_WANT_KEY_TYPE_RAW_DATA 1 - -#endif /* PSA_CRYPTO_ADJUST_AUTO_ENABLED_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_dependencies.h b/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_dependencies.h deleted file mode 100644 index 92e9c4de2..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_dependencies.h +++ /dev/null @@ -1,51 +0,0 @@ -/** - * \file psa/crypto_adjust_config_dependencies.h - * \brief Adjust PSA configuration by resolving some dependencies. - * - * This is an internal header. Do not include it directly. - * - * See docs/proposed/psa-conditional-inclusion-c.md. - * If the Mbed TLS implementation of a cryptographic mechanism A depends on a - * cryptographic mechanism B then if the cryptographic mechanism A is enabled - * and not accelerated enable B. Note that if A is enabled and accelerated, it - * is not necessary to enable B for A support. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_ADJUST_CONFIG_DEPENDENCIES_H -#define PSA_CRYPTO_ADJUST_CONFIG_DEPENDENCIES_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include psa/crypto_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -#if (defined(PSA_WANT_ALG_TLS12_PRF) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_TLS12_PRF)) || \ - (defined(PSA_WANT_ALG_TLS12_PSK_TO_MS) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_TLS12_PSK_TO_MS)) || \ - (defined(PSA_WANT_ALG_HKDF) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF)) || \ - (defined(PSA_WANT_ALG_HKDF_EXTRACT) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF_EXTRACT)) || \ - (defined(PSA_WANT_ALG_HKDF_EXPAND) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_HKDF_EXPAND)) || \ - (defined(PSA_WANT_ALG_PBKDF2_HMAC) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_PBKDF2_HMAC)) -#define PSA_WANT_ALG_HMAC 1 -#define PSA_WANT_KEY_TYPE_HMAC 1 -#endif - -#if (defined(PSA_WANT_ALG_PBKDF2_AES_CMAC_PRF_128) && \ - !defined(MBEDTLS_PSA_ACCEL_ALG_PBKDF2_AES_CMAC_PRF_128)) -#define PSA_WANT_KEY_TYPE_AES 1 -#define PSA_WANT_ALG_CMAC 1 -#endif - -#endif /* PSA_CRYPTO_ADJUST_CONFIG_DEPENDENCIES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_key_pair_types.h b/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_key_pair_types.h deleted file mode 100644 index cec39e01c..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_key_pair_types.h +++ /dev/null @@ -1,101 +0,0 @@ -/** - * \file psa/crypto_adjust_config_key_pair_types.h - * \brief Adjust PSA configuration for key pair types. - * - * This is an internal header. Do not include it directly. - * - * See docs/proposed/psa-conditional-inclusion-c.md. - * - Support non-basic operations in a keypair type implicitly enables basic - * support for that keypair type. - * - Support for a keypair type implicitly enables the corresponding public - * key type. - * - Basic support for a keypair type implicilty enables import/export support - * for that keypair type. Warning: this is implementation-specific (mainly - * for the benefit of testing) and may change in the future! - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_ADJUST_KEYPAIR_TYPES_H -#define PSA_CRYPTO_ADJUST_KEYPAIR_TYPES_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include psa/crypto_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/***************************************************************** - * ANYTHING -> BASIC - ****************************************************************/ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT) || \ - defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT) || \ - defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE) || \ - defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT) || \ - defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT) || \ - defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE) || \ - defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT) || \ - defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT) || \ - defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE) || \ - defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE) -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC 1 -#endif - -/***************************************************************** - * BASIC -> corresponding PUBLIC - ****************************************************************/ - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY 1 -#endif - -/***************************************************************** - * BASIC -> IMPORT+EXPORT - * - * (Implementation-specific, may change in the future.) - ****************************************************************/ - -/* Even though KEY_PAIR symbols' feature several level of support (BASIC, IMPORT, - * EXPORT, GENERATE, DERIVE) we're not planning to have support only for BASIC - * without IMPORT/EXPORT since these last 2 features are strongly used in tests. - * In general it is allowed to include more feature than what is strictly - * requested. - * As a consequence IMPORT and EXPORT features will be automatically enabled - * as soon as the BASIC one is. */ -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#endif - -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT 1 -#endif - -#endif /* PSA_CRYPTO_ADJUST_KEYPAIR_TYPES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_synonyms.h b/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_synonyms.h deleted file mode 100644 index 54b116f43..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_adjust_config_synonyms.h +++ /dev/null @@ -1,49 +0,0 @@ -/** - * \file psa/crypto_adjust_config_synonyms.h - * \brief Adjust PSA configuration: enable quasi-synonyms - * - * This is an internal header. Do not include it directly. - * - * When two features require almost the same code, we automatically enable - * both when either one is requested, to reduce the combinatorics of - * possible configurations. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_ADJUST_CONFIG_SYNONYMS_H -#define PSA_CRYPTO_ADJUST_CONFIG_SYNONYMS_H - -#if !defined(MBEDTLS_CONFIG_FILES_READ) -#error "Do not include psa/crypto_adjust_*.h manually! This can lead to problems, " \ - "up to and including runtime errors such as buffer overflows. " \ - "If you're trying to fix a complaint from check_config.h, just remove " \ - "it from your configuration file: since Mbed TLS 3.0, it is included " \ - "automatically at the right point." -#endif /* */ - -/****************************************************************/ -/* De facto synonyms */ -/****************************************************************/ - -#if defined(PSA_WANT_ALG_ECDSA_ANY) && !defined(PSA_WANT_ALG_ECDSA) -#define PSA_WANT_ALG_ECDSA PSA_WANT_ALG_ECDSA_ANY -#elif !defined(PSA_WANT_ALG_ECDSA_ANY) && defined(PSA_WANT_ALG_ECDSA) -#define PSA_WANT_ALG_ECDSA_ANY PSA_WANT_ALG_ECDSA -#endif - -#if defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN_RAW) && !defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN) -#define PSA_WANT_ALG_RSA_PKCS1V15_SIGN PSA_WANT_ALG_RSA_PKCS1V15_SIGN_RAW -#elif !defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN_RAW) && defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN) -#define PSA_WANT_ALG_RSA_PKCS1V15_SIGN_RAW PSA_WANT_ALG_RSA_PKCS1V15_SIGN -#endif - -#if defined(PSA_WANT_ALG_RSA_PSS_ANY_SALT) && !defined(PSA_WANT_ALG_RSA_PSS) -#define PSA_WANT_ALG_RSA_PSS PSA_WANT_ALG_RSA_PSS_ANY_SALT -#elif !defined(PSA_WANT_ALG_RSA_PSS_ANY_SALT) && defined(PSA_WANT_ALG_RSA_PSS) -#define PSA_WANT_ALG_RSA_PSS_ANY_SALT PSA_WANT_ALG_RSA_PSS -#endif - -#endif /* PSA_CRYPTO_ADJUST_CONFIG_SYNONYMS_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_composites.h b/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_composites.h deleted file mode 100644 index c14f5dd11..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_composites.h +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Context structure declaration of the Mbed TLS software-based PSA drivers - * called through the PSA Crypto driver dispatch layer. - * This file contains the context structures of those algorithms which need to - * rely on other algorithms, i.e. are 'composite' algorithms. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the Mbed TLS software-based PSA drivers. The - * definitions of these objects are then used by crypto_struct.h to define the - * implementation-defined types of PSA multi-part state objects. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_BUILTIN_COMPOSITES_H -#define PSA_CRYPTO_BUILTIN_COMPOSITES_H -#include "mbedtls/private_access.h" - -#include - -#include "mbedtls/cmac.h" -#if defined(MBEDTLS_PSA_BUILTIN_ALG_GCM) -#include "mbedtls/gcm.h" -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_CCM) -#include "mbedtls/ccm.h" -#endif -#include "mbedtls/chachapoly.h" - -/* - * MAC multi-part operation definitions. - */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_CMAC) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_HMAC) -#define MBEDTLS_PSA_BUILTIN_MAC -#endif - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_HMAC) || defined(PSA_CRYPTO_DRIVER_TEST) -typedef struct { - /** The HMAC algorithm in use */ - psa_algorithm_t MBEDTLS_PRIVATE(alg); - /** The hash context. */ - struct psa_hash_operation_s hash_ctx; - /** The HMAC part of the context. */ - uint8_t MBEDTLS_PRIVATE(opad)[PSA_HMAC_MAX_HASH_BLOCK_SIZE]; -} mbedtls_psa_hmac_operation_t; - -#define MBEDTLS_PSA_HMAC_OPERATION_INIT { 0, PSA_HASH_OPERATION_INIT, { 0 } } -#endif /* MBEDTLS_PSA_BUILTIN_ALG_HMAC */ - -typedef struct { - psa_algorithm_t MBEDTLS_PRIVATE(alg); - union { - unsigned MBEDTLS_PRIVATE(dummy); /* Make the union non-empty even with no supported algorithms. */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_HMAC) || defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_psa_hmac_operation_t MBEDTLS_PRIVATE(hmac); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_HMAC */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_CMAC) || defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_cipher_context_t MBEDTLS_PRIVATE(cmac); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_CMAC */ - } MBEDTLS_PRIVATE(ctx); -} mbedtls_psa_mac_operation_t; - -#define MBEDTLS_PSA_MAC_OPERATION_INIT { 0, { 0 } } - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_GCM) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CCM) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305) -#define MBEDTLS_PSA_BUILTIN_AEAD 1 -#endif - -/* Context structure for the Mbed TLS AEAD implementation. */ -typedef struct { - psa_algorithm_t MBEDTLS_PRIVATE(alg); - psa_key_type_t MBEDTLS_PRIVATE(key_type); - - unsigned int MBEDTLS_PRIVATE(is_encrypt) : 1; - - uint8_t MBEDTLS_PRIVATE(tag_length); - - union { - unsigned dummy; /* Enable easier initializing of the union. */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_CCM) - mbedtls_ccm_context MBEDTLS_PRIVATE(ccm); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_CCM */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_GCM) - mbedtls_gcm_context MBEDTLS_PRIVATE(gcm); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_GCM */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305) - mbedtls_chachapoly_context MBEDTLS_PRIVATE(chachapoly); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305 */ - - } ctx; - -} mbedtls_psa_aead_operation_t; - -#define MBEDTLS_PSA_AEAD_OPERATION_INIT { 0, 0, 0, 0, { 0 } } - -#include "mbedtls/ecdsa.h" - -/* Context structure for the Mbed TLS interruptible sign hash implementation. */ -typedef struct { -#if (defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA)) && \ - defined(MBEDTLS_ECP_RESTARTABLE) - mbedtls_ecdsa_context *MBEDTLS_PRIVATE(ctx); - mbedtls_ecdsa_restart_ctx MBEDTLS_PRIVATE(restart_ctx); - - uint32_t MBEDTLS_PRIVATE(num_ops); - - size_t MBEDTLS_PRIVATE(coordinate_bytes); - psa_algorithm_t MBEDTLS_PRIVATE(alg); - mbedtls_md_type_t MBEDTLS_PRIVATE(md_alg); - uint8_t MBEDTLS_PRIVATE(hash)[PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS)]; - size_t MBEDTLS_PRIVATE(hash_length); - -#else - /* Make the struct non-empty if algs not supported. */ - unsigned MBEDTLS_PRIVATE(dummy); - -#endif /* defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || - * defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA) && - * defined( MBEDTLS_ECP_RESTARTABLE ) */ -} mbedtls_psa_sign_hash_interruptible_operation_t; - -#if (defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA)) && \ - defined(MBEDTLS_ECP_RESTARTABLE) -#define MBEDTLS_PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { { 0 }, { 0 }, 0, 0, 0, 0, 0, 0 } -#else -#define MBEDTLS_PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { 0 } -#endif - -/* Context structure for the Mbed TLS interruptible verify hash - * implementation.*/ -typedef struct { -#if (defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA)) && \ - defined(MBEDTLS_ECP_RESTARTABLE) - - mbedtls_ecdsa_context *MBEDTLS_PRIVATE(ctx); - mbedtls_ecdsa_restart_ctx MBEDTLS_PRIVATE(restart_ctx); - - uint32_t MBEDTLS_PRIVATE(num_ops); - - uint8_t MBEDTLS_PRIVATE(hash)[PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS)]; - size_t MBEDTLS_PRIVATE(hash_length); - - mbedtls_mpi MBEDTLS_PRIVATE(r); - mbedtls_mpi MBEDTLS_PRIVATE(s); - -#else - /* Make the struct non-empty if algs not supported. */ - unsigned MBEDTLS_PRIVATE(dummy); - -#endif /* defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || - * defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA) && - * defined( MBEDTLS_ECP_RESTARTABLE ) */ - -} mbedtls_psa_verify_hash_interruptible_operation_t; - -#if (defined(MBEDTLS_PSA_BUILTIN_ALG_ECDSA) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_DETERMINISTIC_ECDSA)) && \ - defined(MBEDTLS_ECP_RESTARTABLE) -#define MBEDTLS_VERIFY_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { { 0 }, { 0 }, 0, 0, 0, 0, { 0 }, \ - { 0 } } -#else -#define MBEDTLS_VERIFY_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { 0 } -#endif - - -/* EC-JPAKE operation definitions */ - -#include "mbedtls/ecjpake.h" - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_JPAKE) -#define MBEDTLS_PSA_BUILTIN_PAKE 1 -#endif - -/* Note: the format for mbedtls_ecjpake_read/write function has an extra - * length byte for each step, plus an extra 3 bytes for ECParameters in the - * server's 2nd round. */ -#define MBEDTLS_PSA_JPAKE_BUFFER_SIZE ((3 + 1 + 65 + 1 + 65 + 1 + 32) * 2) - -typedef struct { - psa_algorithm_t MBEDTLS_PRIVATE(alg); - - uint8_t *MBEDTLS_PRIVATE(password); - size_t MBEDTLS_PRIVATE(password_len); -#if defined(MBEDTLS_PSA_BUILTIN_ALG_JPAKE) - mbedtls_ecjpake_role MBEDTLS_PRIVATE(role); - uint8_t MBEDTLS_PRIVATE(buffer[MBEDTLS_PSA_JPAKE_BUFFER_SIZE]); - size_t MBEDTLS_PRIVATE(buffer_length); - size_t MBEDTLS_PRIVATE(buffer_offset); -#endif - /* Context structure for the Mbed TLS EC-JPAKE implementation. */ - union { - unsigned int MBEDTLS_PRIVATE(dummy); -#if defined(MBEDTLS_PSA_BUILTIN_ALG_JPAKE) - mbedtls_ecjpake_context MBEDTLS_PRIVATE(jpake); -#endif - } MBEDTLS_PRIVATE(ctx); - -} mbedtls_psa_pake_operation_t; - -#define MBEDTLS_PSA_PAKE_OPERATION_INIT { { 0 } } - -#endif /* PSA_CRYPTO_BUILTIN_COMPOSITES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_key_derivation.h b/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_key_derivation.h deleted file mode 100644 index 6b91ae73f..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_key_derivation.h +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Context structure declaration of the Mbed TLS software-based PSA drivers - * called through the PSA Crypto driver dispatch layer. - * This file contains the context structures of key derivation algorithms - * which need to rely on other algorithms. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the Mbed TLS software-based PSA drivers. The - * definitions of these objects are then used by crypto_struct.h to define the - * implementation-defined types of PSA multi-part state objects. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_BUILTIN_KEY_DERIVATION_H -#define PSA_CRYPTO_BUILTIN_KEY_DERIVATION_H -#include "mbedtls/private_access.h" - -#include - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXTRACT) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXPAND) -typedef struct { - uint8_t *MBEDTLS_PRIVATE(info); - size_t MBEDTLS_PRIVATE(info_length); -#if PSA_HASH_MAX_SIZE > 0xff -#error "PSA_HASH_MAX_SIZE does not fit in uint8_t" -#endif - uint8_t MBEDTLS_PRIVATE(offset_in_block); - uint8_t MBEDTLS_PRIVATE(block_number); - unsigned int MBEDTLS_PRIVATE(state) : 2; - unsigned int MBEDTLS_PRIVATE(info_set) : 1; - uint8_t MBEDTLS_PRIVATE(output_block)[PSA_HASH_MAX_SIZE]; - uint8_t MBEDTLS_PRIVATE(prk)[PSA_HASH_MAX_SIZE]; - struct psa_mac_operation_s MBEDTLS_PRIVATE(hmac); -} psa_hkdf_key_derivation_t; -#endif /* MBEDTLS_PSA_BUILTIN_ALG_HKDF || - MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXTRACT || - MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXPAND */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_ECJPAKE_TO_PMS) -typedef struct { - uint8_t MBEDTLS_PRIVATE(data)[PSA_TLS12_ECJPAKE_TO_PMS_DATA_SIZE]; -} psa_tls12_ecjpake_to_pms_t; -#endif /* MBEDTLS_PSA_BUILTIN_ALG_TLS12_ECJPAKE_TO_PMS */ - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_PRF) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS) -typedef enum { - PSA_TLS12_PRF_STATE_INIT, /* no input provided */ - PSA_TLS12_PRF_STATE_SEED_SET, /* seed has been set */ - PSA_TLS12_PRF_STATE_OTHER_KEY_SET, /* other key has been set - optional */ - PSA_TLS12_PRF_STATE_KEY_SET, /* key has been set */ - PSA_TLS12_PRF_STATE_LABEL_SET, /* label has been set */ - PSA_TLS12_PRF_STATE_OUTPUT /* output has been started */ -} psa_tls12_prf_key_derivation_state_t; - -typedef struct psa_tls12_prf_key_derivation_s { -#if PSA_HASH_MAX_SIZE > 0xff -#error "PSA_HASH_MAX_SIZE does not fit in uint8_t" -#endif - - /* Indicates how many bytes in the current HMAC block have - * not yet been read by the user. */ - uint8_t MBEDTLS_PRIVATE(left_in_block); - - /* The 1-based number of the block. */ - uint8_t MBEDTLS_PRIVATE(block_number); - - psa_tls12_prf_key_derivation_state_t MBEDTLS_PRIVATE(state); - - uint8_t *MBEDTLS_PRIVATE(secret); - size_t MBEDTLS_PRIVATE(secret_length); - uint8_t *MBEDTLS_PRIVATE(seed); - size_t MBEDTLS_PRIVATE(seed_length); - uint8_t *MBEDTLS_PRIVATE(label); - size_t MBEDTLS_PRIVATE(label_length); -#if defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS) - uint8_t *MBEDTLS_PRIVATE(other_secret); - size_t MBEDTLS_PRIVATE(other_secret_length); -#endif /* MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS */ - - uint8_t MBEDTLS_PRIVATE(Ai)[PSA_HASH_MAX_SIZE]; - - /* `HMAC_hash( prk, A( i ) + seed )` in the notation of RFC 5246, Sect. 5. */ - uint8_t MBEDTLS_PRIVATE(output_block)[PSA_HASH_MAX_SIZE]; -} psa_tls12_prf_key_derivation_t; -#endif /* MBEDTLS_PSA_BUILTIN_ALG_TLS12_PRF) || - * MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS */ -#if defined(PSA_HAVE_SOFT_PBKDF2) -typedef enum { - PSA_PBKDF2_STATE_INIT, /* no input provided */ - PSA_PBKDF2_STATE_INPUT_COST_SET, /* input cost has been set */ - PSA_PBKDF2_STATE_SALT_SET, /* salt has been set */ - PSA_PBKDF2_STATE_PASSWORD_SET, /* password has been set */ - PSA_PBKDF2_STATE_OUTPUT /* output has been started */ -} psa_pbkdf2_key_derivation_state_t; - -typedef struct { - psa_pbkdf2_key_derivation_state_t MBEDTLS_PRIVATE(state); - uint64_t MBEDTLS_PRIVATE(input_cost); - uint8_t *MBEDTLS_PRIVATE(salt); - size_t MBEDTLS_PRIVATE(salt_length); - uint8_t MBEDTLS_PRIVATE(password)[PSA_HMAC_MAX_HASH_BLOCK_SIZE]; - size_t MBEDTLS_PRIVATE(password_length); - uint8_t MBEDTLS_PRIVATE(output_block)[PSA_HASH_MAX_SIZE]; - uint8_t MBEDTLS_PRIVATE(bytes_used); - uint32_t MBEDTLS_PRIVATE(block_number); -} psa_pbkdf2_key_derivation_t; -#endif /* PSA_HAVE_SOFT_PBKDF2 */ - -#endif /* PSA_CRYPTO_BUILTIN_KEY_DERIVATION_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_primitives.h b/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_primitives.h deleted file mode 100644 index 98ab4d333..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_builtin_primitives.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Context structure declaration of the Mbed TLS software-based PSA drivers - * called through the PSA Crypto driver dispatch layer. - * This file contains the context structures of those algorithms which do not - * rely on other algorithms, i.e. are 'primitive' algorithms. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the Mbed TLS software-based PSA drivers. The - * definitions of these objects are then used by crypto_struct.h to define the - * implementation-defined types of PSA multi-part state objects. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_BUILTIN_PRIMITIVES_H -#define PSA_CRYPTO_BUILTIN_PRIMITIVES_H -#include "mbedtls/private_access.h" - -#include - -/* - * Hash multi-part operation definitions. - */ - -#include "mbedtls/md5.h" -#include "mbedtls/ripemd160.h" -#include "mbedtls/sha1.h" -#include "mbedtls/sha256.h" -#include "mbedtls/sha512.h" -#include "mbedtls/sha3.h" - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_MD5) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_RIPEMD160) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_1) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_224) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_256) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_384) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_512) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_224) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_256) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_384) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_512) -#define MBEDTLS_PSA_BUILTIN_HASH -#endif - -typedef struct { - psa_algorithm_t MBEDTLS_PRIVATE(alg); - union { - unsigned dummy; /* Make the union non-empty even with no supported algorithms. */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_MD5) - mbedtls_md5_context md5; -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_RIPEMD160) - mbedtls_ripemd160_context ripemd160; -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_1) - mbedtls_sha1_context sha1; -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_256) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_224) - mbedtls_sha256_context sha256; -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_512) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA_384) - mbedtls_sha512_context sha512; -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_224) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_256) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_384) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_SHA3_512) - mbedtls_sha3_context sha3; -#endif - } MBEDTLS_PRIVATE(ctx); -} mbedtls_psa_hash_operation_t; - -#define MBEDTLS_PSA_HASH_OPERATION_INIT { 0, { 0 } } - -/* - * Cipher multi-part operation definitions. - */ - -#include "mbedtls/cipher.h" - -#if defined(MBEDTLS_PSA_BUILTIN_ALG_STREAM_CIPHER) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CTR) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CFB) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_OFB) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_ECB_NO_PADDING) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CBC_NO_PADDING) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CBC_PKCS7) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_CCM_STAR_NO_TAG) -#define MBEDTLS_PSA_BUILTIN_CIPHER 1 -#endif - -typedef struct { - /* Context structure for the Mbed TLS cipher implementation. */ - psa_algorithm_t MBEDTLS_PRIVATE(alg); - uint8_t MBEDTLS_PRIVATE(iv_length); - uint8_t MBEDTLS_PRIVATE(block_length); - union { - unsigned int MBEDTLS_PRIVATE(dummy); - mbedtls_cipher_context_t MBEDTLS_PRIVATE(cipher); - } MBEDTLS_PRIVATE(ctx); -} mbedtls_psa_cipher_operation_t; - -#define MBEDTLS_PSA_CIPHER_OPERATION_INIT { 0, 0, 0, { 0 } } - -#endif /* PSA_CRYPTO_BUILTIN_PRIMITIVES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_compat.h b/android/mbedtls/arm64-v8a/include/psa/crypto_compat.h deleted file mode 100644 index 2a226c01a..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_compat.h +++ /dev/null @@ -1,230 +0,0 @@ -/** - * \file psa/crypto_compat.h - * - * \brief PSA cryptography module: Backward compatibility aliases - * - * This header declares alternative names for macro and functions. - * New application code should not use these names. - * These names may be removed in a future version of Mbed TLS. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_COMPAT_H -#define PSA_CRYPTO_COMPAT_H - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * To support both openless APIs and psa_open_key() temporarily, define - * psa_key_handle_t to be equal to mbedtls_svc_key_id_t. Do not mark the - * type and its utility macros and functions deprecated yet. This will be done - * in a subsequent phase. - */ -typedef mbedtls_svc_key_id_t psa_key_handle_t; - -#define PSA_KEY_HANDLE_INIT MBEDTLS_SVC_KEY_ID_INIT - -/** Check whether a handle is null. - * - * \param handle Handle - * - * \return Non-zero if the handle is null, zero otherwise. - */ -static inline int psa_key_handle_is_null(psa_key_handle_t handle) -{ - return mbedtls_svc_key_id_is_null(handle); -} - -/** Open a handle to an existing persistent key. - * - * Open a handle to a persistent key. A key is persistent if it was created - * with a lifetime other than #PSA_KEY_LIFETIME_VOLATILE. A persistent key - * always has a nonzero key identifier, set with psa_set_key_id() when - * creating the key. Implementations may provide additional pre-provisioned - * keys that can be opened with psa_open_key(). Such keys have an application - * key identifier in the vendor range, as documented in the description of - * #psa_key_id_t. - * - * The application must eventually close the handle with psa_close_key() or - * psa_destroy_key() to release associated resources. If the application dies - * without calling one of these functions, the implementation should perform - * the equivalent of a call to psa_close_key(). - * - * Some implementations permit an application to open the same key multiple - * times. If this is successful, each call to psa_open_key() will return a - * different key handle. - * - * \note This API is not part of the PSA Cryptography API Release 1.0.0 - * specification. It was defined in the 1.0 Beta 3 version of the - * specification but was removed in the 1.0.0 released version. This API is - * kept for the time being to not break applications relying on it. It is not - * deprecated yet but will be in the near future. - * - * \note Applications that rely on opening a key multiple times will not be - * portable to implementations that only permit a single key handle to be - * opened. See also :ref:\`key-handles\`. - * - * - * \param key The persistent identifier of the key. - * \param[out] handle On success, a handle to the key. - * - * \retval #PSA_SUCCESS - * Success. The application can now use the value of `*handle` - * to access the key. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY - * The implementation does not have sufficient resources to open the - * key. This can be due to reaching an implementation limit on the - * number of open keys, the number of open key handles, or available - * memory. - * \retval #PSA_ERROR_DOES_NOT_EXIST - * There is no persistent key with key identifier \p key. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p key is not a valid persistent key identifier. - * \retval #PSA_ERROR_NOT_PERMITTED - * The specified key exists, but the application does not have the - * permission to access it. Note that this specification does not - * define any way to create such a key, but it may be possible - * through implementation-specific means. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_open_key(mbedtls_svc_key_id_t key, - psa_key_handle_t *handle); - -/** Close a key handle. - * - * If the handle designates a volatile key, this will destroy the key material - * and free all associated resources, just like psa_destroy_key(). - * - * If this is the last open handle to a persistent key, then closing the handle - * will free all resources associated with the key in volatile memory. The key - * data in persistent storage is not affected and can be opened again later - * with a call to psa_open_key(). - * - * Closing the key handle makes the handle invalid, and the key handle - * must not be used again by the application. - * - * \note This API is not part of the PSA Cryptography API Release 1.0.0 - * specification. It was defined in the 1.0 Beta 3 version of the - * specification but was removed in the 1.0.0 released version. This API is - * kept for the time being to not break applications relying on it. It is not - * deprecated yet but will be in the near future. - * - * \note If the key handle was used to set up an active - * :ref:\`multipart operation \`, then closing the - * key handle can cause the multipart operation to fail. Applications should - * maintain the key handle until after the multipart operation has finished. - * - * \param handle The key handle to close. - * If this is \c 0, do nothing and return \c PSA_SUCCESS. - * - * \retval #PSA_SUCCESS - * \p handle was a valid handle or \c 0. It is now closed. - * \retval #PSA_ERROR_INVALID_HANDLE - * \p handle is not a valid handle nor \c 0. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_close_key(psa_key_handle_t handle); - -/** \addtogroup attributes - * @{ - */ - -#if !defined(MBEDTLS_DEPRECATED_REMOVED) -/** Custom Diffie-Hellman group. - * - * Mbed TLS does not support custom DH groups. - * - * \deprecated This value is not useful, so this macro will be removed in - * a future version of the library. - */ -#define PSA_DH_FAMILY_CUSTOM \ - ((psa_dh_family_t) MBEDTLS_DEPRECATED_NUMERIC_CONSTANT(0x7e)) - -/** - * \brief Set domain parameters for a key. - * - * \deprecated Mbed TLS no longer supports any domain parameters. - * This function only does the equivalent of - * psa_set_key_type() and will be removed in a future version - * of the library. - * - * \param[in,out] attributes Attribute structure where \p type will be set. - * \param type Key type (a \c PSA_KEY_TYPE_XXX value). - * \param[in] data Ignored. - * \param data_length Must be 0. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - */ -static inline psa_status_t MBEDTLS_DEPRECATED psa_set_key_domain_parameters( - psa_key_attributes_t *attributes, - psa_key_type_t type, const uint8_t *data, size_t data_length) -{ - (void) data; - if (data_length != 0) { - return PSA_ERROR_NOT_SUPPORTED; - } - psa_set_key_type(attributes, type); - return PSA_SUCCESS; -} - -/** - * \brief Get domain parameters for a key. - * - * \deprecated Mbed TLS no longer supports any domain parameters. - * This function alwaya has an empty output and will be - * removed in a future version of the library. - - * \param[in] attributes Ignored. - * \param[out] data Ignored. - * \param data_size Ignored. - * \param[out] data_length Set to 0. - * - * \retval #PSA_SUCCESS \emptydescription - */ -static inline psa_status_t MBEDTLS_DEPRECATED psa_get_key_domain_parameters( - const psa_key_attributes_t *attributes, - uint8_t *data, size_t data_size, size_t *data_length) -{ - (void) attributes; - (void) data; - (void) data_size; - *data_length = 0; - return PSA_SUCCESS; -} - -/** Safe output buffer size for psa_get_key_domain_parameters(). - * - */ -#define PSA_KEY_DOMAIN_PARAMETERS_SIZE(key_type, key_bits) \ - MBEDTLS_DEPRECATED_NUMERIC_CONSTANT(1u) -#endif /* MBEDTLS_DEPRECATED_REMOVED */ - -/**@}*/ - -#ifdef __cplusplus -} -#endif - -#endif /* PSA_CRYPTO_COMPAT_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_config.h b/android/mbedtls/arm64-v8a/include/psa/crypto_config.h deleted file mode 100644 index e0a8d6060..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_config.h +++ /dev/null @@ -1,145 +0,0 @@ -/** - * \file psa/crypto_config.h - * \brief PSA crypto configuration options (set of defines) - * - */ -#if defined(MBEDTLS_PSA_CRYPTO_CONFIG) -/** - * When #MBEDTLS_PSA_CRYPTO_CONFIG is enabled in mbedtls_config.h, - * this file determines which cryptographic mechanisms are enabled - * through the PSA Cryptography API (\c psa_xxx() functions). - * - * To enable a cryptographic mechanism, uncomment the definition of - * the corresponding \c PSA_WANT_xxx preprocessor symbol. - * To disable a cryptographic mechanism, comment out the definition of - * the corresponding \c PSA_WANT_xxx preprocessor symbol. - * The names of cryptographic mechanisms correspond to values - * defined in psa/crypto_values.h, with the prefix \c PSA_WANT_ instead - * of \c PSA_. - * - * Note that many cryptographic mechanisms involve two symbols: one for - * the key type (\c PSA_WANT_KEY_TYPE_xxx) and one for the algorithm - * (\c PSA_WANT_ALG_xxx). Mechanisms with additional parameters may involve - * additional symbols. - */ -#else -/** - * When \c MBEDTLS_PSA_CRYPTO_CONFIG is disabled in mbedtls_config.h, - * this file is not used, and cryptographic mechanisms are supported - * through the PSA API if and only if they are supported through the - * mbedtls_xxx API. - */ -#endif -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_CONFIG_H -#define PSA_CRYPTO_CONFIG_H - -#define PSA_WANT_ALG_CBC_NO_PADDING 1 -#define PSA_WANT_ALG_CBC_PKCS7 1 -#define PSA_WANT_ALG_CCM 1 -#define PSA_WANT_ALG_CCM_STAR_NO_TAG 1 -#define PSA_WANT_ALG_CMAC 1 -#define PSA_WANT_ALG_CFB 1 -#define PSA_WANT_ALG_CHACHA20_POLY1305 1 -#define PSA_WANT_ALG_CTR 1 -#define PSA_WANT_ALG_DETERMINISTIC_ECDSA 1 -#define PSA_WANT_ALG_ECB_NO_PADDING 1 -#define PSA_WANT_ALG_ECDH 1 -#define PSA_WANT_ALG_FFDH 1 -#define PSA_WANT_ALG_ECDSA 1 -#define PSA_WANT_ALG_JPAKE 1 -#define PSA_WANT_ALG_GCM 1 -#define PSA_WANT_ALG_HKDF 1 -#define PSA_WANT_ALG_HKDF_EXTRACT 1 -#define PSA_WANT_ALG_HKDF_EXPAND 1 -#define PSA_WANT_ALG_HMAC 1 -#define PSA_WANT_ALG_MD5 1 -#define PSA_WANT_ALG_OFB 1 -#define PSA_WANT_ALG_PBKDF2_HMAC 1 -#define PSA_WANT_ALG_PBKDF2_AES_CMAC_PRF_128 1 -#define PSA_WANT_ALG_RIPEMD160 1 -#define PSA_WANT_ALG_RSA_OAEP 1 -#define PSA_WANT_ALG_RSA_PKCS1V15_CRYPT 1 -#define PSA_WANT_ALG_RSA_PKCS1V15_SIGN 1 -#define PSA_WANT_ALG_RSA_PSS 1 -#define PSA_WANT_ALG_SHA_1 1 -#define PSA_WANT_ALG_SHA_224 1 -#define PSA_WANT_ALG_SHA_256 1 -#define PSA_WANT_ALG_SHA_384 1 -#define PSA_WANT_ALG_SHA_512 1 -#define PSA_WANT_ALG_SHA3_224 1 -#define PSA_WANT_ALG_SHA3_256 1 -#define PSA_WANT_ALG_SHA3_384 1 -#define PSA_WANT_ALG_SHA3_512 1 -#define PSA_WANT_ALG_STREAM_CIPHER 1 -#define PSA_WANT_ALG_TLS12_PRF 1 -#define PSA_WANT_ALG_TLS12_PSK_TO_MS 1 -#define PSA_WANT_ALG_TLS12_ECJPAKE_TO_PMS 1 - -#define PSA_WANT_ECC_BRAINPOOL_P_R1_256 1 -#define PSA_WANT_ECC_BRAINPOOL_P_R1_384 1 -#define PSA_WANT_ECC_BRAINPOOL_P_R1_512 1 -#define PSA_WANT_ECC_MONTGOMERY_255 1 -#define PSA_WANT_ECC_MONTGOMERY_448 1 -#define PSA_WANT_ECC_SECP_K1_192 1 -#define PSA_WANT_ECC_SECP_K1_256 1 -#define PSA_WANT_ECC_SECP_R1_192 1 -#define PSA_WANT_ECC_SECP_R1_224 1 -/* For secp256r1, consider enabling #MBEDTLS_PSA_P256M_DRIVER_ENABLED - * (see the description in mbedtls/mbedtls_config.h for details). */ -#define PSA_WANT_ECC_SECP_R1_256 1 -#define PSA_WANT_ECC_SECP_R1_384 1 -#define PSA_WANT_ECC_SECP_R1_521 1 - -#define PSA_WANT_DH_RFC7919_2048 1 -#define PSA_WANT_DH_RFC7919_3072 1 -#define PSA_WANT_DH_RFC7919_4096 1 -#define PSA_WANT_DH_RFC7919_6144 1 -#define PSA_WANT_DH_RFC7919_8192 1 - -#define PSA_WANT_KEY_TYPE_DERIVE 1 -#define PSA_WANT_KEY_TYPE_PASSWORD 1 -#define PSA_WANT_KEY_TYPE_PASSWORD_HASH 1 -#define PSA_WANT_KEY_TYPE_HMAC 1 -#define PSA_WANT_KEY_TYPE_AES 1 -#define PSA_WANT_KEY_TYPE_ARIA 1 -#define PSA_WANT_KEY_TYPE_CAMELLIA 1 -#define PSA_WANT_KEY_TYPE_CHACHA20 1 -#define PSA_WANT_KEY_TYPE_DES 1 -//#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR 1 /* Deprecated */ -#define PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY 1 -#define PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY 1 -#define PSA_WANT_KEY_TYPE_RAW_DATA 1 -//#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR 1 /* Deprecated */ -#define PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY 1 - -/* - * The following symbols extend and deprecate the legacy - * PSA_WANT_KEY_TYPE_xxx_KEY_PAIR ones. They include the usage of that key in - * the name's suffix. "_USE" is the most generic and it can be used to describe - * a generic suport, whereas other ones add more features on top of that and - * they are more specific. - */ -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE 1 -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE 1 - -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE 1 -//#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE 1 /* Not supported */ - -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT 1 -#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE 1 -//#define PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE 1 /* Not supported */ - -#endif /* PSA_CRYPTO_CONFIG_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_common.h b/android/mbedtls/arm64-v8a/include/psa/crypto_driver_common.h deleted file mode 100644 index cc11d3b9a..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_common.h +++ /dev/null @@ -1,44 +0,0 @@ -/** - * \file psa/crypto_driver_common.h - * \brief Definitions for all PSA crypto drivers - * - * This file contains common definitions shared by all PSA crypto drivers. - * Do not include it directly: instead, include the header file(s) for - * the type(s) of driver that you are implementing. For example, if - * you are writing a dynamically registered driver for a secure element, - * include `psa/crypto_se_driver.h`. - * - * This file is part of the PSA Crypto Driver Model, containing functions for - * driver developers to implement to enable hardware to be called in a - * standardized way by a PSA Cryptographic API implementation. The functions - * comprising the driver model, which driver authors implement, are not - * intended to be called by application developers. - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef PSA_CRYPTO_DRIVER_COMMON_H -#define PSA_CRYPTO_DRIVER_COMMON_H - -#include -#include - -/* Include type definitions (psa_status_t, psa_algorithm_t, - * psa_key_type_t, etc.) and macros to build and analyze values - * of these types. */ -#include "crypto_types.h" -#include "crypto_values.h" -/* Include size definitions which are used to size some arrays in operation - * structures. */ -#include - -/** For encrypt-decrypt functions, whether the operation is an encryption - * or a decryption. */ -typedef enum { - PSA_CRYPTO_DRIVER_DECRYPT, - PSA_CRYPTO_DRIVER_ENCRYPT -} psa_encrypt_or_decrypt_t; - -#endif /* PSA_CRYPTO_DRIVER_COMMON_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_composites.h b/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_composites.h deleted file mode 100644 index d717c5190..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_composites.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Declaration of context structures for use with the PSA driver wrapper - * interface. This file contains the context structures for 'composite' - * operations, i.e. those operations which need to make use of other operations - * from the primitives (crypto_driver_contexts_primitives.h) - * - * Warning: This file will be auto-generated in the future. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the PSA drivers included in the cryptographic - * library. The definitions of these objects are then used by crypto_struct.h - * to define the implementation-defined types of PSA multi-part state objects. - */ -/* Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_DRIVER_CONTEXTS_COMPOSITES_H -#define PSA_CRYPTO_DRIVER_CONTEXTS_COMPOSITES_H - -#include "psa/crypto_driver_common.h" - -/* Include the context structure definitions for the Mbed TLS software drivers */ -#include "psa/crypto_builtin_composites.h" - -/* Include the context structure definitions for those drivers that were - * declared during the autogeneration process. */ - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) -#include -#endif - -#if defined(PSA_CRYPTO_DRIVER_TEST) -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) && \ - defined(LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_MAC) -typedef libtestdriver1_mbedtls_psa_mac_operation_t - mbedtls_transparent_test_driver_mac_operation_t; -typedef libtestdriver1_mbedtls_psa_mac_operation_t - mbedtls_opaque_test_driver_mac_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_MAC_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_MAC_OPERATION_INIT -#define MBEDTLS_OPAQUE_TEST_DRIVER_MAC_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_MAC_OPERATION_INIT - -#else -typedef mbedtls_psa_mac_operation_t - mbedtls_transparent_test_driver_mac_operation_t; -typedef mbedtls_psa_mac_operation_t - mbedtls_opaque_test_driver_mac_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_MAC_OPERATION_INIT \ - MBEDTLS_PSA_MAC_OPERATION_INIT -#define MBEDTLS_OPAQUE_TEST_DRIVER_MAC_OPERATION_INIT \ - MBEDTLS_PSA_MAC_OPERATION_INIT - -#endif /* MBEDTLS_TEST_LIBTESTDRIVER1 && LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_MAC */ - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) && \ - defined(LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_AEAD) -typedef libtestdriver1_mbedtls_psa_aead_operation_t - mbedtls_transparent_test_driver_aead_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_AEAD_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_AEAD_OPERATION_INIT -#else -typedef mbedtls_psa_aead_operation_t - mbedtls_transparent_test_driver_aead_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_AEAD_OPERATION_INIT \ - MBEDTLS_PSA_AEAD_OPERATION_INIT - -#endif /* MBEDTLS_TEST_LIBTESTDRIVER1 && LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_AEAD */ - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) && \ - defined(LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_PAKE) - -typedef libtestdriver1_mbedtls_psa_pake_operation_t - mbedtls_transparent_test_driver_pake_operation_t; -typedef libtestdriver1_mbedtls_psa_pake_operation_t - mbedtls_opaque_test_driver_pake_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_PAKE_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_PAKE_OPERATION_INIT -#define MBEDTLS_OPAQUE_TEST_DRIVER_PAKE_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_PAKE_OPERATION_INIT - -#else -typedef mbedtls_psa_pake_operation_t - mbedtls_transparent_test_driver_pake_operation_t; -typedef mbedtls_psa_pake_operation_t - mbedtls_opaque_test_driver_pake_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_PAKE_OPERATION_INIT \ - MBEDTLS_PSA_PAKE_OPERATION_INIT -#define MBEDTLS_OPAQUE_TEST_DRIVER_PAKE_OPERATION_INIT \ - MBEDTLS_PSA_PAKE_OPERATION_INIT - -#endif /* MBEDTLS_TEST_LIBTESTDRIVER1 && LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_PAKE */ - -#endif /* PSA_CRYPTO_DRIVER_TEST */ - -/* Define the context to be used for an operation that is executed through the - * PSA Driver wrapper layer as the union of all possible driver's contexts. - * - * The union members are the driver's context structures, and the member names - * are formatted as `'drivername'_ctx`. This allows for procedural generation - * of both this file and the content of psa_crypto_driver_wrappers.h */ - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_mac_operation_t mbedtls_ctx; -#if defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_transparent_test_driver_mac_operation_t transparent_test_driver_ctx; - mbedtls_opaque_test_driver_mac_operation_t opaque_test_driver_ctx; -#endif -} psa_driver_mac_context_t; - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_aead_operation_t mbedtls_ctx; -#if defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_transparent_test_driver_aead_operation_t transparent_test_driver_ctx; -#endif -} psa_driver_aead_context_t; - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_sign_hash_interruptible_operation_t mbedtls_ctx; -} psa_driver_sign_hash_interruptible_context_t; - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_verify_hash_interruptible_operation_t mbedtls_ctx; -} psa_driver_verify_hash_interruptible_context_t; - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_pake_operation_t mbedtls_ctx; -#if defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_transparent_test_driver_pake_operation_t transparent_test_driver_ctx; - mbedtls_opaque_test_driver_pake_operation_t opaque_test_driver_ctx; -#endif -} psa_driver_pake_context_t; - -#endif /* PSA_CRYPTO_DRIVER_CONTEXTS_COMPOSITES_H */ -/* End of automatically generated file. */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_key_derivation.h b/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_key_derivation.h deleted file mode 100644 index 21190515c..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_key_derivation.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Declaration of context structures for use with the PSA driver wrapper - * interface. This file contains the context structures for key derivation - * operations. - * - * Warning: This file will be auto-generated in the future. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the PSA drivers included in the cryptographic - * library. The definitions of these objects are then used by crypto_struct.h - * to define the implementation-defined types of PSA multi-part state objects. - */ -/* Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_DRIVER_CONTEXTS_KEY_DERIVATION_H -#define PSA_CRYPTO_DRIVER_CONTEXTS_KEY_DERIVATION_H - -#include "psa/crypto_driver_common.h" - -/* Include the context structure definitions for the Mbed TLS software drivers */ -#include "psa/crypto_builtin_key_derivation.h" - -/* Include the context structure definitions for those drivers that were - * declared during the autogeneration process. */ - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ -#if defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXTRACT) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_HKDF_EXPAND) - psa_hkdf_key_derivation_t MBEDTLS_PRIVATE(hkdf); -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_PRF) || \ - defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_PSK_TO_MS) - psa_tls12_prf_key_derivation_t MBEDTLS_PRIVATE(tls12_prf); -#endif -#if defined(MBEDTLS_PSA_BUILTIN_ALG_TLS12_ECJPAKE_TO_PMS) - psa_tls12_ecjpake_to_pms_t MBEDTLS_PRIVATE(tls12_ecjpake_to_pms); -#endif -#if defined(PSA_HAVE_SOFT_PBKDF2) - psa_pbkdf2_key_derivation_t MBEDTLS_PRIVATE(pbkdf2); -#endif -} psa_driver_key_derivation_context_t; - -#endif /* PSA_CRYPTO_DRIVER_CONTEXTS_KEY_DERIVATION_H */ -/* End of automatically generated file. */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_primitives.h b/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_primitives.h deleted file mode 100644 index c90a5fbe7..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_driver_contexts_primitives.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Declaration of context structures for use with the PSA driver wrapper - * interface. This file contains the context structures for 'primitive' - * operations, i.e. those operations which do not rely on other contexts. - * - * Warning: This file will be auto-generated in the future. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * \note This header and its content are not part of the Mbed TLS API and - * applications must not depend on it. Its main purpose is to define the - * multi-part state objects of the PSA drivers included in the cryptographic - * library. The definitions of these objects are then used by crypto_struct.h - * to define the implementation-defined types of PSA multi-part state objects. - */ -/* Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_DRIVER_CONTEXTS_PRIMITIVES_H -#define PSA_CRYPTO_DRIVER_CONTEXTS_PRIMITIVES_H - -#include "psa/crypto_driver_common.h" - -/* Include the context structure definitions for the Mbed TLS software drivers */ -#include "psa/crypto_builtin_primitives.h" - -/* Include the context structure definitions for those drivers that were - * declared during the autogeneration process. */ - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) -#include -#endif - -#if defined(PSA_CRYPTO_DRIVER_TEST) - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) && \ - defined(LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_CIPHER) -typedef libtestdriver1_mbedtls_psa_cipher_operation_t - mbedtls_transparent_test_driver_cipher_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_CIPHER_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_CIPHER_OPERATION_INIT -#else -typedef mbedtls_psa_cipher_operation_t - mbedtls_transparent_test_driver_cipher_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_CIPHER_OPERATION_INIT \ - MBEDTLS_PSA_CIPHER_OPERATION_INIT -#endif /* MBEDTLS_TEST_LIBTESTDRIVER1 && - LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_CIPHER */ - -#if defined(MBEDTLS_TEST_LIBTESTDRIVER1) && \ - defined(LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_HASH) -typedef libtestdriver1_mbedtls_psa_hash_operation_t - mbedtls_transparent_test_driver_hash_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_HASH_OPERATION_INIT \ - LIBTESTDRIVER1_MBEDTLS_PSA_HASH_OPERATION_INIT -#else -typedef mbedtls_psa_hash_operation_t - mbedtls_transparent_test_driver_hash_operation_t; - -#define MBEDTLS_TRANSPARENT_TEST_DRIVER_HASH_OPERATION_INIT \ - MBEDTLS_PSA_HASH_OPERATION_INIT -#endif /* MBEDTLS_TEST_LIBTESTDRIVER1 && - LIBTESTDRIVER1_MBEDTLS_PSA_BUILTIN_HASH */ - -typedef struct { - unsigned int initialised : 1; - mbedtls_transparent_test_driver_cipher_operation_t ctx; -} mbedtls_opaque_test_driver_cipher_operation_t; - -#define MBEDTLS_OPAQUE_TEST_DRIVER_CIPHER_OPERATION_INIT \ - { 0, MBEDTLS_TRANSPARENT_TEST_DRIVER_CIPHER_OPERATION_INIT } - -#endif /* PSA_CRYPTO_DRIVER_TEST */ - -/* Define the context to be used for an operation that is executed through the - * PSA Driver wrapper layer as the union of all possible driver's contexts. - * - * The union members are the driver's context structures, and the member names - * are formatted as `'drivername'_ctx`. This allows for procedural generation - * of both this file and the content of psa_crypto_driver_wrappers.h */ - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_hash_operation_t mbedtls_ctx; -#if defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_transparent_test_driver_hash_operation_t test_driver_ctx; -#endif -} psa_driver_hash_context_t; - -typedef union { - unsigned dummy; /* Make sure this union is always non-empty */ - mbedtls_psa_cipher_operation_t mbedtls_ctx; -#if defined(PSA_CRYPTO_DRIVER_TEST) - mbedtls_transparent_test_driver_cipher_operation_t transparent_test_driver_ctx; - mbedtls_opaque_test_driver_cipher_operation_t opaque_test_driver_ctx; -#endif -} psa_driver_cipher_context_t; - -#endif /* PSA_CRYPTO_DRIVER_CONTEXTS_PRIMITIVES_H */ -/* End of automatically generated file. */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_extra.h b/android/mbedtls/arm64-v8a/include/psa/crypto_extra.h deleted file mode 100644 index a710397a7..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_extra.h +++ /dev/null @@ -1,1977 +0,0 @@ -/** - * \file psa/crypto_extra.h - * - * \brief PSA cryptography module: Mbed TLS vendor extensions - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * This file is reserved for vendor-specific definitions. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_EXTRA_H -#define PSA_CRYPTO_EXTRA_H -#include "mbedtls/private_access.h" - -#include "crypto_types.h" -#include "crypto_compat.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* UID for secure storage seed */ -#define PSA_CRYPTO_ITS_RANDOM_SEED_UID 0xFFFFFF52 - -/* See mbedtls_config.h for definition */ -#if !defined(MBEDTLS_PSA_KEY_SLOT_COUNT) -#define MBEDTLS_PSA_KEY_SLOT_COUNT 32 -#endif - -/* If the size of static key slots is not explicitly defined by the user, then - * set it to the maximum between PSA_EXPORT_KEY_PAIR_OR_PUBLIC_MAX_SIZE and - * PSA_CIPHER_MAX_KEY_LENGTH. - * See mbedtls_config.h for the definition. */ -#if !defined(MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE) -#define MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE \ - ((PSA_EXPORT_KEY_PAIR_OR_PUBLIC_MAX_SIZE > PSA_CIPHER_MAX_KEY_LENGTH) ? \ - PSA_EXPORT_KEY_PAIR_OR_PUBLIC_MAX_SIZE : PSA_CIPHER_MAX_KEY_LENGTH) -#endif /* !MBEDTLS_PSA_STATIC_KEY_SLOT_BUFFER_SIZE*/ - -/** \addtogroup attributes - * @{ - */ - -/** \brief Declare the enrollment algorithm for a key. - * - * An operation on a key may indifferently use the algorithm set with - * psa_set_key_algorithm() or with this function. - * - * \param[out] attributes The attribute structure to write to. - * \param alg2 A second algorithm that the key may be used - * for, in addition to the algorithm set with - * psa_set_key_algorithm(). - * - * \warning Setting an enrollment algorithm is not recommended, because - * using the same key with different algorithms can allow some - * attacks based on arithmetic relations between different - * computations made with the same key, or can escalate harmless - * side channels into exploitable ones. Use this function only - * if it is necessary to support a protocol for which it has been - * verified that the usage of the key with multiple algorithms - * is safe. - */ -static inline void psa_set_key_enrollment_algorithm( - psa_key_attributes_t *attributes, - psa_algorithm_t alg2) -{ - attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(alg2) = alg2; -} - -/** Retrieve the enrollment algorithm policy from key attributes. - * - * \param[in] attributes The key attribute structure to query. - * - * \return The enrollment algorithm stored in the attribute structure. - */ -static inline psa_algorithm_t psa_get_key_enrollment_algorithm( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(alg2); -} - -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) - -/** Retrieve the slot number where a key is stored. - * - * A slot number is only defined for keys that are stored in a secure - * element. - * - * This information is only useful if the secure element is not entirely - * managed through the PSA Cryptography API. It is up to the secure - * element driver to decide how PSA slot numbers map to any other interface - * that the secure element may have. - * - * \param[in] attributes The key attribute structure to query. - * \param[out] slot_number On success, the slot number containing the key. - * - * \retval #PSA_SUCCESS - * The key is located in a secure element, and \p *slot_number - * indicates the slot number that contains it. - * \retval #PSA_ERROR_NOT_PERMITTED - * The caller is not permitted to query the slot number. - * Mbed TLS currently does not return this error. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The key is not located in a secure element. - */ -psa_status_t psa_get_key_slot_number( - const psa_key_attributes_t *attributes, - psa_key_slot_number_t *slot_number); - -/** Choose the slot number where a key is stored. - * - * This function declares a slot number in the specified attribute - * structure. - * - * A slot number is only meaningful for keys that are stored in a secure - * element. It is up to the secure element driver to decide how PSA slot - * numbers map to any other interface that the secure element may have. - * - * \note Setting a slot number in key attributes for a key creation can - * cause the following errors when creating the key: - * - #PSA_ERROR_NOT_SUPPORTED if the selected secure element does - * not support choosing a specific slot number. - * - #PSA_ERROR_NOT_PERMITTED if the caller is not permitted to - * choose slot numbers in general or to choose this specific slot. - * - #PSA_ERROR_INVALID_ARGUMENT if the chosen slot number is not - * valid in general or not valid for this specific key. - * - #PSA_ERROR_ALREADY_EXISTS if there is already a key in the - * selected slot. - * - * \param[out] attributes The attribute structure to write to. - * \param slot_number The slot number to set. - */ -static inline void psa_set_key_slot_number( - psa_key_attributes_t *attributes, - psa_key_slot_number_t slot_number) -{ - attributes->MBEDTLS_PRIVATE(has_slot_number) = 1; - attributes->MBEDTLS_PRIVATE(slot_number) = slot_number; -} - -/** Remove the slot number attribute from a key attribute structure. - * - * This function undoes the action of psa_set_key_slot_number(). - * - * \param[out] attributes The attribute structure to write to. - */ -static inline void psa_clear_key_slot_number( - psa_key_attributes_t *attributes) -{ - attributes->MBEDTLS_PRIVATE(has_slot_number) = 0; -} - -/** Register a key that is already present in a secure element. - * - * The key must be located in a secure element designated by the - * lifetime field in \p attributes, in the slot set with - * psa_set_key_slot_number() in the attribute structure. - * This function makes the key available through the key identifier - * specified in \p attributes. - * - * \param[in] attributes The attributes of the existing key. - * - The lifetime must be a persistent lifetime - * in a secure element. Volatile lifetimes are - * not currently supported. - * - The key identifier must be in the valid - * range for persistent keys. - * - The key type and size must be specified and - * must be consistent with the key material - * in the secure element. - * - * \retval #PSA_SUCCESS - * The key was successfully registered. - * Note that depending on the design of the driver, this may or may - * not guarantee that a key actually exists in the designated slot - * and is compatible with the specified attributes. - * \retval #PSA_ERROR_ALREADY_EXISTS - * There is already a key with the identifier specified in - * \p attributes. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The secure element driver for the specified lifetime does not - * support registering a key. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The identifier in \p attributes is invalid, namely the identifier is - * not in the user range, or - * \p attributes specifies a lifetime which is not located - * in a secure element, or no slot number is specified in \p attributes, - * or the specified slot number is not valid. - * \retval #PSA_ERROR_NOT_PERMITTED - * The caller is not authorized to register the specified key slot. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t mbedtls_psa_register_se_key( - const psa_key_attributes_t *attributes); - -#endif /* MBEDTLS_PSA_CRYPTO_SE_C */ - -/**@}*/ - -/** - * \brief Library deinitialization. - * - * This function clears all data associated with the PSA layer, - * including the whole key store. - * This function is not thread safe, it wipes every key slot regardless of - * state and reader count. It should only be called when no slot is in use. - * - * This is an Mbed TLS extension. - */ -void mbedtls_psa_crypto_free(void); - -/** \brief Statistics about - * resource consumption related to the PSA keystore. - * - * \note The content of this structure is not part of the stable API and ABI - * of Mbed TLS and may change arbitrarily from version to version. - */ -typedef struct mbedtls_psa_stats_s { - /** Number of slots containing key material for a volatile key. */ - size_t MBEDTLS_PRIVATE(volatile_slots); - /** Number of slots containing key material for a key which is in - * internal persistent storage. */ - size_t MBEDTLS_PRIVATE(persistent_slots); - /** Number of slots containing a reference to a key in a - * secure element. */ - size_t MBEDTLS_PRIVATE(external_slots); - /** Number of slots which are occupied, but do not contain - * key material yet. */ - size_t MBEDTLS_PRIVATE(half_filled_slots); - /** Number of slots that contain cache data. */ - size_t MBEDTLS_PRIVATE(cache_slots); - /** Number of slots that are not used for anything. */ - size_t MBEDTLS_PRIVATE(empty_slots); - /** Number of slots that are locked. */ - size_t MBEDTLS_PRIVATE(locked_slots); - /** Largest key id value among open keys in internal persistent storage. */ - psa_key_id_t MBEDTLS_PRIVATE(max_open_internal_key_id); - /** Largest key id value among open keys in secure elements. */ - psa_key_id_t MBEDTLS_PRIVATE(max_open_external_key_id); -} mbedtls_psa_stats_t; - -/** \brief Get statistics about - * resource consumption related to the PSA keystore. - * - * \note When Mbed TLS is built as part of a service, with isolation - * between the application and the keystore, the service may or - * may not expose this function. - */ -void mbedtls_psa_get_stats(mbedtls_psa_stats_t *stats); - -/** - * \brief Inject an initial entropy seed for the random generator into - * secure storage. - * - * This function injects data to be used as a seed for the random generator - * used by the PSA Crypto implementation. On devices that lack a trusted - * entropy source (preferably a hardware random number generator), - * the Mbed PSA Crypto implementation uses this value to seed its - * random generator. - * - * On devices without a trusted entropy source, this function must be - * called exactly once in the lifetime of the device. On devices with - * a trusted entropy source, calling this function is optional. - * In all cases, this function may only be called before calling any - * other function in the PSA Crypto API, including psa_crypto_init(). - * - * When this function returns successfully, it populates a file in - * persistent storage. Once the file has been created, this function - * can no longer succeed. - * - * If any error occurs, this function does not change the system state. - * You can call this function again after correcting the reason for the - * error if possible. - * - * \warning This function **can** fail! Callers MUST check the return status. - * - * \warning If you use this function, you should use it as part of a - * factory provisioning process. The value of the injected seed - * is critical to the security of the device. It must be - * *secret*, *unpredictable* and (statistically) *unique per device*. - * You should be generate it randomly using a cryptographically - * secure random generator seeded from trusted entropy sources. - * You should transmit it securely to the device and ensure - * that its value is not leaked or stored anywhere beyond the - * needs of transmitting it from the point of generation to - * the call of this function, and erase all copies of the value - * once this function returns. - * - * This is an Mbed TLS extension. - * - * \note This function is only available on the following platforms: - * * If the compile-time option MBEDTLS_PSA_INJECT_ENTROPY is enabled. - * Note that you must provide compatible implementations of - * mbedtls_nv_seed_read and mbedtls_nv_seed_write. - * * In a client-server integration of PSA Cryptography, on the client side, - * if the server supports this feature. - * \param[in] seed Buffer containing the seed value to inject. - * \param[in] seed_size Size of the \p seed buffer. - * The size of the seed in bytes must be greater - * or equal to both #MBEDTLS_ENTROPY_BLOCK_SIZE - * and the value of \c MBEDTLS_ENTROPY_MIN_PLATFORM - * in `library/entropy_poll.h` in the Mbed TLS source - * code. - * It must be less or equal to - * #MBEDTLS_ENTROPY_MAX_SEED_SIZE. - * - * \retval #PSA_SUCCESS - * The seed value was injected successfully. The random generator - * of the PSA Crypto implementation is now ready for use. - * You may now call psa_crypto_init() and use the PSA Crypto - * implementation. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p seed_size is out of range. - * \retval #PSA_ERROR_STORAGE_FAILURE - * There was a failure reading or writing from storage. - * \retval #PSA_ERROR_NOT_PERMITTED - * The library has already been initialized. It is no longer - * possible to call this function. - */ -psa_status_t mbedtls_psa_inject_entropy(const uint8_t *seed, - size_t seed_size); - -/** \addtogroup crypto_types - * @{ - */ - -/** DSA public key. - * - * The import and export format is the - * representation of the public key `y = g^x mod p` as a big-endian byte - * string. The length of the byte string is the length of the base prime `p` - * in bytes. - */ -#define PSA_KEY_TYPE_DSA_PUBLIC_KEY ((psa_key_type_t) 0x4002) - -/** DSA key pair (private and public key). - * - * The import and export format is the - * representation of the private key `x` as a big-endian byte string. The - * length of the byte string is the private key size in bytes (leading zeroes - * are not stripped). - * - * Deterministic DSA key derivation with psa_generate_derived_key follows - * FIPS 186-4 §B.1.2: interpret the byte string as integer - * in big-endian order. Discard it if it is not in the range - * [0, *N* - 2] where *N* is the boundary of the private key domain - * (the prime *p* for Diffie-Hellman, the subprime *q* for DSA, - * or the order of the curve's base point for ECC). - * Add 1 to the resulting integer and use this as the private key *x*. - * - */ -#define PSA_KEY_TYPE_DSA_KEY_PAIR ((psa_key_type_t) 0x7002) - -/** Whether a key type is a DSA key (pair or public-only). */ -#define PSA_KEY_TYPE_IS_DSA(type) \ - (PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(type) == PSA_KEY_TYPE_DSA_PUBLIC_KEY) - -#define PSA_ALG_DSA_BASE ((psa_algorithm_t) 0x06000400) -/** DSA signature with hashing. - * - * This is the signature scheme defined by FIPS 186-4, - * with a random per-message secret number (*k*). - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding DSA signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_DSA(hash_alg) \ - (PSA_ALG_DSA_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -#define PSA_ALG_DETERMINISTIC_DSA_BASE ((psa_algorithm_t) 0x06000500) -#define PSA_ALG_DSA_DETERMINISTIC_FLAG PSA_ALG_ECDSA_DETERMINISTIC_FLAG -/** Deterministic DSA signature with hashing. - * - * This is the deterministic variant defined by RFC 6979 of - * the signature scheme defined by FIPS 186-4. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding DSA signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_DETERMINISTIC_DSA(hash_alg) \ - (PSA_ALG_DETERMINISTIC_DSA_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -#define PSA_ALG_IS_DSA(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK & ~PSA_ALG_DSA_DETERMINISTIC_FLAG) == \ - PSA_ALG_DSA_BASE) -#define PSA_ALG_DSA_IS_DETERMINISTIC(alg) \ - (((alg) & PSA_ALG_DSA_DETERMINISTIC_FLAG) != 0) -#define PSA_ALG_IS_DETERMINISTIC_DSA(alg) \ - (PSA_ALG_IS_DSA(alg) && PSA_ALG_DSA_IS_DETERMINISTIC(alg)) -#define PSA_ALG_IS_RANDOMIZED_DSA(alg) \ - (PSA_ALG_IS_DSA(alg) && !PSA_ALG_DSA_IS_DETERMINISTIC(alg)) - - -/* We need to expand the sample definition of this macro from - * the API definition. */ -#undef PSA_ALG_IS_VENDOR_HASH_AND_SIGN -#define PSA_ALG_IS_VENDOR_HASH_AND_SIGN(alg) \ - PSA_ALG_IS_DSA(alg) - -/**@}*/ - -/** \addtogroup attributes - * @{ - */ - -/** PAKE operation stages. */ -#define PSA_PAKE_OPERATION_STAGE_SETUP 0 -#define PSA_PAKE_OPERATION_STAGE_COLLECT_INPUTS 1 -#define PSA_PAKE_OPERATION_STAGE_COMPUTATION 2 - -/**@}*/ - - -/** \defgroup psa_external_rng External random generator - * @{ - */ - -#if defined(MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG) -/** External random generator function, implemented by the platform. - * - * When the compile-time option #MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG is enabled, - * this function replaces Mbed TLS's entropy and DRBG modules for all - * random generation triggered via PSA crypto interfaces. - * - * \note This random generator must deliver random numbers with cryptographic - * quality and high performance. It must supply unpredictable numbers - * with a uniform distribution. The implementation of this function - * is responsible for ensuring that the random generator is seeded - * with sufficient entropy. If you have a hardware TRNG which is slow - * or delivers non-uniform output, declare it as an entropy source - * with mbedtls_entropy_add_source() instead of enabling this option. - * - * \param[in,out] context Pointer to the random generator context. - * This is all-bits-zero on the first call - * and preserved between successive calls. - * \param[out] output Output buffer. On success, this buffer - * contains random data with a uniform - * distribution. - * \param output_size The size of the \p output buffer in bytes. - * \param[out] output_length On success, set this value to \p output_size. - * - * \retval #PSA_SUCCESS - * Success. The output buffer contains \p output_size bytes of - * cryptographic-quality random data, and \c *output_length is - * set to \p output_size. - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY - * The random generator requires extra entropy and there is no - * way to obtain entropy under current environment conditions. - * This error should not happen under normal circumstances since - * this function is responsible for obtaining as much entropy as - * it needs. However implementations of this function may return - * #PSA_ERROR_INSUFFICIENT_ENTROPY if there is no way to obtain - * entropy without blocking indefinitely. - * \retval #PSA_ERROR_HARDWARE_FAILURE - * A failure of the random generator hardware that isn't covered - * by #PSA_ERROR_INSUFFICIENT_ENTROPY. - */ -psa_status_t mbedtls_psa_external_get_random( - mbedtls_psa_external_random_context_t *context, - uint8_t *output, size_t output_size, size_t *output_length); -#endif /* MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG */ - -/**@}*/ - -/** \defgroup psa_builtin_keys Built-in keys - * @{ - */ - -/** The minimum value for a key identifier that is built into the - * implementation. - * - * The range of key identifiers from #MBEDTLS_PSA_KEY_ID_BUILTIN_MIN - * to #MBEDTLS_PSA_KEY_ID_BUILTIN_MAX within the range from - * #PSA_KEY_ID_VENDOR_MIN and #PSA_KEY_ID_VENDOR_MAX and must not intersect - * with any other set of implementation-chosen key identifiers. - * - * This value is part of the library's API since changing it would invalidate - * the values of built-in key identifiers in applications. - */ -#define MBEDTLS_PSA_KEY_ID_BUILTIN_MIN ((psa_key_id_t) 0x7fff0000) - -/** The maximum value for a key identifier that is built into the - * implementation. - * - * See #MBEDTLS_PSA_KEY_ID_BUILTIN_MIN for more information. - */ -#define MBEDTLS_PSA_KEY_ID_BUILTIN_MAX ((psa_key_id_t) 0x7fffefff) - -/** A slot number identifying a key in a driver. - * - * Values of this type are used to identify built-in keys. - */ -typedef uint64_t psa_drv_slot_number_t; - -#if defined(MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS) -/** Test whether a key identifier belongs to the builtin key range. - * - * \param key_id Key identifier to test. - * - * \retval 1 - * The key identifier is a builtin key identifier. - * \retval 0 - * The key identifier is not a builtin key identifier. - */ -static inline int psa_key_id_is_builtin(psa_key_id_t key_id) -{ - return (key_id >= MBEDTLS_PSA_KEY_ID_BUILTIN_MIN) && - (key_id <= MBEDTLS_PSA_KEY_ID_BUILTIN_MAX); -} - -/** Platform function to obtain the location and slot number of a built-in key. - * - * An application-specific implementation of this function must be provided if - * #MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS is enabled. This would typically be provided - * as part of a platform's system image. - * - * #MBEDTLS_SVC_KEY_ID_GET_KEY_ID(\p key_id) needs to be in the range from - * #MBEDTLS_PSA_KEY_ID_BUILTIN_MIN to #MBEDTLS_PSA_KEY_ID_BUILTIN_MAX. - * - * In a multi-application configuration - * (\c MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER is defined), - * this function should check that #MBEDTLS_SVC_KEY_ID_GET_OWNER_ID(\p key_id) - * is allowed to use the given key. - * - * \param key_id The key ID for which to retrieve the - * location and slot attributes. - * \param[out] lifetime On success, the lifetime associated with the key - * corresponding to \p key_id. Lifetime is a - * combination of which driver contains the key, - * and with what persistence level the key is - * intended to be used. If the platform - * implementation does not contain specific - * information about the intended key persistence - * level, the persistence level may be reported as - * #PSA_KEY_PERSISTENCE_DEFAULT. - * \param[out] slot_number On success, the slot number known to the driver - * registered at the lifetime location reported - * through \p lifetime which corresponds to the - * requested built-in key. - * - * \retval #PSA_SUCCESS - * The requested key identifier designates a built-in key. - * In a multi-application configuration, the requested owner - * is allowed to access it. - * \retval #PSA_ERROR_DOES_NOT_EXIST - * The requested key identifier is not a built-in key which is known - * to this function. If a key exists in the key storage with this - * identifier, the data from the storage will be used. - * \return (any other error) - * Any other error is propagated to the function that requested the key. - * Common errors include: - * - #PSA_ERROR_NOT_PERMITTED: the key exists but the requested owner - * is not allowed to access it. - */ -psa_status_t mbedtls_psa_platform_get_builtin_key( - mbedtls_svc_key_id_t key_id, - psa_key_lifetime_t *lifetime, - psa_drv_slot_number_t *slot_number); -#endif /* MBEDTLS_PSA_CRYPTO_BUILTIN_KEYS */ - -/** @} */ - -/** \defgroup psa_crypto_client Functions defined by a client provider - * - * The functions in this group are meant to be implemented by providers of - * the PSA Crypto client interface. They are provided by the library when - * #MBEDTLS_PSA_CRYPTO_C is enabled. - * - * \note All functions in this group are experimental, as using - * alternative client interface providers is experimental. - * - * @{ - */ - -/** Check if PSA is capable of handling the specified hash algorithm. - * - * This means that PSA core was built with the corresponding PSA_WANT_ALG_xxx - * set and that psa_crypto_init has already been called. - * - * \note When using the built-in version of the PSA core (i.e. - * #MBEDTLS_PSA_CRYPTO_C is set), for now, this function only checks - * the state of the driver subsystem, not the algorithm. - * This might be improved in the future. - * - * \param hash_alg The hash algorithm. - * - * \return 1 if the PSA can handle \p hash_alg, 0 otherwise. - */ -int psa_can_do_hash(psa_algorithm_t hash_alg); - -/** - * Tell if PSA is ready for this cipher. - * - * \note When using the built-in version of the PSA core (i.e. - * #MBEDTLS_PSA_CRYPTO_C is set), for now, this function only checks - * the state of the driver subsystem, not the key type and algorithm. - * This might be improved in the future. - * - * \param key_type The key type. - * \param cipher_alg The cipher algorithm. - * - * \return 1 if the PSA can handle \p cipher_alg, 0 otherwise. - */ -int psa_can_do_cipher(psa_key_type_t key_type, psa_algorithm_t cipher_alg); - -/**@}*/ - -/** \addtogroup crypto_types - * @{ - */ - -#define PSA_ALG_CATEGORY_PAKE ((psa_algorithm_t) 0x0a000000) - -/** Whether the specified algorithm is a password-authenticated key exchange. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a password-authenticated key exchange (PAKE) - * algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_PAKE(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_PAKE) - -/** The Password-authenticated key exchange by juggling (J-PAKE) algorithm. - * - * This is J-PAKE as defined by RFC 8236, instantiated with the following - * parameters: - * - * - The group can be either an elliptic curve or defined over a finite field. - * - Schnorr NIZK proof as defined by RFC 8235 and using the same group as the - * J-PAKE algorithm. - * - A cryptographic hash function. - * - * To select these parameters and set up the cipher suite, call these functions - * in any order: - * - * \code - * psa_pake_cs_set_algorithm(cipher_suite, PSA_ALG_JPAKE); - * psa_pake_cs_set_primitive(cipher_suite, - * PSA_PAKE_PRIMITIVE(type, family, bits)); - * psa_pake_cs_set_hash(cipher_suite, hash); - * \endcode - * - * For more information on how to set a specific curve or field, refer to the - * documentation of the individual \c PSA_PAKE_PRIMITIVE_TYPE_XXX constants. - * - * After initializing a J-PAKE operation, call - * - * \code - * psa_pake_setup(operation, cipher_suite); - * psa_pake_set_user(operation, ...); - * psa_pake_set_peer(operation, ...); - * psa_pake_set_password_key(operation, ...); - * \endcode - * - * The password is provided as a key. This can be the password text itself, - * in an agreed character encoding, or some value derived from the password - * as required by a higher level protocol. - * - * (The implementation converts the key material to a number as described in - * Section 2.3.8 of _SEC 1: Elliptic Curve Cryptography_ - * (https://www.secg.org/sec1-v2.pdf), before reducing it modulo \c q. Here - * \c q is order of the group defined by the primitive set in the cipher suite. - * The \c psa_pake_set_password_key() function returns an error if the result - * of the reduction is 0.) - * - * The key exchange flow for J-PAKE is as follows: - * -# To get the first round data that needs to be sent to the peer, call - * \code - * // Get g1 - * psa_pake_output(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Get the ZKP public key for x1 - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Get the ZKP proof for x1 - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * // Get g2 - * psa_pake_output(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Get the ZKP public key for x2 - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Get the ZKP proof for x2 - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * \endcode - * -# To provide the first round data received from the peer to the operation, - * call - * \code - * // Set g3 - * psa_pake_input(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Set the ZKP public key for x3 - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Set the ZKP proof for x3 - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * // Set g4 - * psa_pake_input(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Set the ZKP public key for x4 - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Set the ZKP proof for x4 - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * \endcode - * -# To get the second round data that needs to be sent to the peer, call - * \code - * // Get A - * psa_pake_output(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Get ZKP public key for x2*s - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Get ZKP proof for x2*s - * psa_pake_output(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * \endcode - * -# To provide the second round data received from the peer to the operation, - * call - * \code - * // Set B - * psa_pake_input(operation, #PSA_PAKE_STEP_KEY_SHARE, ...); - * // Set ZKP public key for x4*s - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PUBLIC, ...); - * // Set ZKP proof for x4*s - * psa_pake_input(operation, #PSA_PAKE_STEP_ZK_PROOF, ...); - * \endcode - * -# To access the shared secret call - * \code - * // Get Ka=Kb=K - * psa_pake_get_implicit_key() - * \endcode - * - * For more information consult the documentation of the individual - * \c PSA_PAKE_STEP_XXX constants. - * - * At this point there is a cryptographic guarantee that only the authenticated - * party who used the same password is able to compute the key. But there is no - * guarantee that the peer is the party it claims to be and was able to do so. - * - * That is, the authentication is only implicit (the peer is not authenticated - * at this point, and no action should be taken that assume that they are - like - * for example accessing restricted files). - * - * To make the authentication explicit there are various methods, see Section 5 - * of RFC 8236 for two examples. - * - * \note The JPAKE implementation has the following limitations: - * - The only supported primitive is ECC on the curve secp256r1, i.e. - * `PSA_PAKE_PRIMITIVE(PSA_PAKE_PRIMITIVE_TYPE_ECC, - * PSA_ECC_FAMILY_SECP_R1, 256)`. - * - The only supported hash algorithm is SHA-256, i.e. - * `PSA_ALG_SHA_256`. - * - When using the built-in implementation, the user ID and the peer ID - * must be `"client"` (6-byte string) and `"server"` (6-byte string), - * or the other way round. - * Third-party drivers may or may not have this limitation. - * - */ -#define PSA_ALG_JPAKE ((psa_algorithm_t) 0x0a000100) - -/** @} */ - -/** \defgroup pake Password-authenticated key exchange (PAKE) - * - * This is a proposed PAKE interface for the PSA Crypto API. It is not part of - * the official PSA Crypto API yet. - * - * \note The content of this section is not part of the stable API and ABI - * of Mbed TLS and may change arbitrarily from version to version. - * Same holds for the corresponding macros #PSA_ALG_CATEGORY_PAKE and - * #PSA_ALG_JPAKE. - * @{ - */ - -/** \brief Encoding of the application role of PAKE - * - * Encodes the application's role in the algorithm is being executed. For more - * information see the documentation of individual \c PSA_PAKE_ROLE_XXX - * constants. - */ -typedef uint8_t psa_pake_role_t; - -/** Encoding of input and output indicators for PAKE. - * - * Some PAKE algorithms need to exchange more data than just a single key share. - * This type is for encoding additional input and output data for such - * algorithms. - */ -typedef uint8_t psa_pake_step_t; - -/** Encoding of the type of the PAKE's primitive. - * - * Values defined by this standard will never be in the range 0x80-0xff. - * Vendors who define additional types must use an encoding in this range. - * - * For more information see the documentation of individual - * \c PSA_PAKE_PRIMITIVE_TYPE_XXX constants. - */ -typedef uint8_t psa_pake_primitive_type_t; - -/** \brief Encoding of the family of the primitive associated with the PAKE. - * - * For more information see the documentation of individual - * \c PSA_PAKE_PRIMITIVE_TYPE_XXX constants. - */ -typedef uint8_t psa_pake_family_t; - -/** \brief Encoding of the primitive associated with the PAKE. - * - * For more information see the documentation of the #PSA_PAKE_PRIMITIVE macro. - */ -typedef uint32_t psa_pake_primitive_t; - -/** A value to indicate no role in a PAKE algorithm. - * This value can be used in a call to psa_pake_set_role() for symmetric PAKE - * algorithms which do not assign roles. - */ -#define PSA_PAKE_ROLE_NONE ((psa_pake_role_t) 0x00) - -/** The first peer in a balanced PAKE. - * - * Although balanced PAKE algorithms are symmetric, some of them needs an - * ordering of peers for the transcript calculations. If the algorithm does not - * need this, both #PSA_PAKE_ROLE_FIRST and #PSA_PAKE_ROLE_SECOND are - * accepted. - */ -#define PSA_PAKE_ROLE_FIRST ((psa_pake_role_t) 0x01) - -/** The second peer in a balanced PAKE. - * - * Although balanced PAKE algorithms are symmetric, some of them needs an - * ordering of peers for the transcript calculations. If the algorithm does not - * need this, either #PSA_PAKE_ROLE_FIRST or #PSA_PAKE_ROLE_SECOND are - * accepted. - */ -#define PSA_PAKE_ROLE_SECOND ((psa_pake_role_t) 0x02) - -/** The client in an augmented PAKE. - * - * Augmented PAKE algorithms need to differentiate between client and server. - */ -#define PSA_PAKE_ROLE_CLIENT ((psa_pake_role_t) 0x11) - -/** The server in an augmented PAKE. - * - * Augmented PAKE algorithms need to differentiate between client and server. - */ -#define PSA_PAKE_ROLE_SERVER ((psa_pake_role_t) 0x12) - -/** The PAKE primitive type indicating the use of elliptic curves. - * - * The values of the \c family and \c bits fields of the cipher suite identify a - * specific elliptic curve, using the same mapping that is used for ECC - * (::psa_ecc_family_t) keys. - * - * (Here \c family means the value returned by psa_pake_cs_get_family() and - * \c bits means the value returned by psa_pake_cs_get_bits().) - * - * Input and output during the operation can involve group elements and scalar - * values: - * -# The format for group elements is the same as for public keys on the - * specific curve would be. For more information, consult the documentation of - * psa_export_public_key(). - * -# The format for scalars is the same as for private keys on the specific - * curve would be. For more information, consult the documentation of - * psa_export_key(). - */ -#define PSA_PAKE_PRIMITIVE_TYPE_ECC ((psa_pake_primitive_type_t) 0x01) - -/** The PAKE primitive type indicating the use of Diffie-Hellman groups. - * - * The values of the \c family and \c bits fields of the cipher suite identify - * a specific Diffie-Hellman group, using the same mapping that is used for - * Diffie-Hellman (::psa_dh_family_t) keys. - * - * (Here \c family means the value returned by psa_pake_cs_get_family() and - * \c bits means the value returned by psa_pake_cs_get_bits().) - * - * Input and output during the operation can involve group elements and scalar - * values: - * -# The format for group elements is the same as for public keys on the - * specific group would be. For more information, consult the documentation of - * psa_export_public_key(). - * -# The format for scalars is the same as for private keys on the specific - * group would be. For more information, consult the documentation of - * psa_export_key(). - */ -#define PSA_PAKE_PRIMITIVE_TYPE_DH ((psa_pake_primitive_type_t) 0x02) - -/** Construct a PAKE primitive from type, family and bit-size. - * - * \param pake_type The type of the primitive - * (value of type ::psa_pake_primitive_type_t). - * \param pake_family The family of the primitive - * (the type and interpretation of this parameter depends - * on \p pake_type, for more information consult the - * documentation of individual ::psa_pake_primitive_type_t - * constants). - * \param pake_bits The bit-size of the primitive - * (Value of type \c size_t. The interpretation - * of this parameter depends on \p pake_family, for more - * information consult the documentation of individual - * ::psa_pake_primitive_type_t constants). - * - * \return The constructed primitive value of type ::psa_pake_primitive_t. - * Return 0 if the requested primitive can't be encoded as - * ::psa_pake_primitive_t. - */ -#define PSA_PAKE_PRIMITIVE(pake_type, pake_family, pake_bits) \ - ((pake_bits & 0xFFFF) != pake_bits) ? 0 : \ - ((psa_pake_primitive_t) (((pake_type) << 24 | \ - (pake_family) << 16) | (pake_bits))) - -/** The key share being sent to or received from the peer. - * - * The format for both input and output at this step is the same as for public - * keys on the group determined by the primitive (::psa_pake_primitive_t) would - * be. - * - * For more information on the format, consult the documentation of - * psa_export_public_key(). - * - * For information regarding how the group is determined, consult the - * documentation #PSA_PAKE_PRIMITIVE. - */ -#define PSA_PAKE_STEP_KEY_SHARE ((psa_pake_step_t) 0x01) - -/** A Schnorr NIZKP public key. - * - * This is the ephemeral public key in the Schnorr Non-Interactive - * Zero-Knowledge Proof (the value denoted by the letter 'V' in RFC 8235). - * - * The format for both input and output at this step is the same as for public - * keys on the group determined by the primitive (::psa_pake_primitive_t) would - * be. - * - * For more information on the format, consult the documentation of - * psa_export_public_key(). - * - * For information regarding how the group is determined, consult the - * documentation #PSA_PAKE_PRIMITIVE. - */ -#define PSA_PAKE_STEP_ZK_PUBLIC ((psa_pake_step_t) 0x02) - -/** A Schnorr NIZKP proof. - * - * This is the proof in the Schnorr Non-Interactive Zero-Knowledge Proof (the - * value denoted by the letter 'r' in RFC 8235). - * - * Both for input and output, the value at this step is an integer less than - * the order of the group selected in the cipher suite. The format depends on - * the group as well: - * - * - For Montgomery curves, the encoding is little endian. - * - For everything else the encoding is big endian (see Section 2.3.8 of - * _SEC 1: Elliptic Curve Cryptography_ at https://www.secg.org/sec1-v2.pdf). - * - * In both cases leading zeroes are allowed as long as the length in bytes does - * not exceed the byte length of the group order. - * - * For information regarding how the group is determined, consult the - * documentation #PSA_PAKE_PRIMITIVE. - */ -#define PSA_PAKE_STEP_ZK_PROOF ((psa_pake_step_t) 0x03) - -/**@}*/ - -/** A sufficient output buffer size for psa_pake_output(). - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_pake_output() will not fail due to an insufficient output buffer - * size. The actual size of the output might be smaller in any given call. - * - * See also #PSA_PAKE_OUTPUT_MAX_SIZE - * - * \param alg A PAKE algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_PAKE(\p alg) is true). - * \param primitive A primitive of type ::psa_pake_primitive_t that is - * compatible with algorithm \p alg. - * \param output_step A value of type ::psa_pake_step_t that is valid for the - * algorithm \p alg. - * \return A sufficient output buffer size for the specified - * PAKE algorithm, primitive, and output step. If the - * PAKE algorithm, primitive, or output step is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_PAKE_OUTPUT_SIZE(alg, primitive, output_step) \ - (alg == PSA_ALG_JPAKE && \ - primitive == PSA_PAKE_PRIMITIVE(PSA_PAKE_PRIMITIVE_TYPE_ECC, \ - PSA_ECC_FAMILY_SECP_R1, 256) ? \ - ( \ - output_step == PSA_PAKE_STEP_KEY_SHARE ? 65 : \ - output_step == PSA_PAKE_STEP_ZK_PUBLIC ? 65 : \ - 32 \ - ) : \ - 0) - -/** A sufficient input buffer size for psa_pake_input(). - * - * The value returned by this macro is guaranteed to be large enough for any - * valid input to psa_pake_input() in an operation with the specified - * parameters. - * - * See also #PSA_PAKE_INPUT_MAX_SIZE - * - * \param alg A PAKE algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_PAKE(\p alg) is true). - * \param primitive A primitive of type ::psa_pake_primitive_t that is - * compatible with algorithm \p alg. - * \param input_step A value of type ::psa_pake_step_t that is valid for the - * algorithm \p alg. - * \return A sufficient input buffer size for the specified - * input, cipher suite and algorithm. If the cipher suite, - * the input type or PAKE algorithm is not recognized, or - * the parameters are incompatible, return 0. - */ -#define PSA_PAKE_INPUT_SIZE(alg, primitive, input_step) \ - (alg == PSA_ALG_JPAKE && \ - primitive == PSA_PAKE_PRIMITIVE(PSA_PAKE_PRIMITIVE_TYPE_ECC, \ - PSA_ECC_FAMILY_SECP_R1, 256) ? \ - ( \ - input_step == PSA_PAKE_STEP_KEY_SHARE ? 65 : \ - input_step == PSA_PAKE_STEP_ZK_PUBLIC ? 65 : \ - 32 \ - ) : \ - 0) - -/** Output buffer size for psa_pake_output() for any of the supported PAKE - * algorithm and primitive suites and output step. - * - * This macro must expand to a compile-time constant integer. - * - * The value of this macro must be at least as large as the largest value - * returned by PSA_PAKE_OUTPUT_SIZE() - * - * See also #PSA_PAKE_OUTPUT_SIZE(\p alg, \p primitive, \p output_step). - */ -#define PSA_PAKE_OUTPUT_MAX_SIZE 65 - -/** Input buffer size for psa_pake_input() for any of the supported PAKE - * algorithm and primitive suites and input step. - * - * This macro must expand to a compile-time constant integer. - * - * The value of this macro must be at least as large as the largest value - * returned by PSA_PAKE_INPUT_SIZE() - * - * See also #PSA_PAKE_INPUT_SIZE(\p alg, \p primitive, \p output_step). - */ -#define PSA_PAKE_INPUT_MAX_SIZE 65 - -/** Returns a suitable initializer for a PAKE cipher suite object of type - * psa_pake_cipher_suite_t. - */ -#define PSA_PAKE_CIPHER_SUITE_INIT { PSA_ALG_NONE, 0, 0, 0, PSA_ALG_NONE } - -/** Returns a suitable initializer for a PAKE operation object of type - * psa_pake_operation_t. - */ -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_PAKE_OPERATION_INIT { 0 } -#else -#define PSA_PAKE_OPERATION_INIT { 0, PSA_ALG_NONE, 0, PSA_PAKE_OPERATION_STAGE_SETUP, \ - { 0 }, { { 0 } } } -#endif - -struct psa_pake_cipher_suite_s { - psa_algorithm_t algorithm; - psa_pake_primitive_type_t type; - psa_pake_family_t family; - uint16_t bits; - psa_algorithm_t hash; -}; - -struct psa_crypto_driver_pake_inputs_s { - uint8_t *MBEDTLS_PRIVATE(password); - size_t MBEDTLS_PRIVATE(password_len); - uint8_t *MBEDTLS_PRIVATE(user); - size_t MBEDTLS_PRIVATE(user_len); - uint8_t *MBEDTLS_PRIVATE(peer); - size_t MBEDTLS_PRIVATE(peer_len); - psa_key_attributes_t MBEDTLS_PRIVATE(attributes); - struct psa_pake_cipher_suite_s MBEDTLS_PRIVATE(cipher_suite); -}; - -typedef enum psa_crypto_driver_pake_step { - PSA_JPAKE_STEP_INVALID = 0, /* Invalid step */ - PSA_JPAKE_X1_STEP_KEY_SHARE = 1, /* Round 1: input/output key share (for ephemeral private key X1).*/ - PSA_JPAKE_X1_STEP_ZK_PUBLIC = 2, /* Round 1: input/output Schnorr NIZKP public key for the X1 key */ - PSA_JPAKE_X1_STEP_ZK_PROOF = 3, /* Round 1: input/output Schnorr NIZKP proof for the X1 key */ - PSA_JPAKE_X2_STEP_KEY_SHARE = 4, /* Round 1: input/output key share (for ephemeral private key X2).*/ - PSA_JPAKE_X2_STEP_ZK_PUBLIC = 5, /* Round 1: input/output Schnorr NIZKP public key for the X2 key */ - PSA_JPAKE_X2_STEP_ZK_PROOF = 6, /* Round 1: input/output Schnorr NIZKP proof for the X2 key */ - PSA_JPAKE_X2S_STEP_KEY_SHARE = 7, /* Round 2: output X2S key (our key) */ - PSA_JPAKE_X2S_STEP_ZK_PUBLIC = 8, /* Round 2: output Schnorr NIZKP public key for the X2S key (our key) */ - PSA_JPAKE_X2S_STEP_ZK_PROOF = 9, /* Round 2: output Schnorr NIZKP proof for the X2S key (our key) */ - PSA_JPAKE_X4S_STEP_KEY_SHARE = 10, /* Round 2: input X4S key (from peer) */ - PSA_JPAKE_X4S_STEP_ZK_PUBLIC = 11, /* Round 2: input Schnorr NIZKP public key for the X4S key (from peer) */ - PSA_JPAKE_X4S_STEP_ZK_PROOF = 12 /* Round 2: input Schnorr NIZKP proof for the X4S key (from peer) */ -} psa_crypto_driver_pake_step_t; - -typedef enum psa_jpake_round { - PSA_JPAKE_FIRST = 0, - PSA_JPAKE_SECOND = 1, - PSA_JPAKE_FINISHED = 2 -} psa_jpake_round_t; - -typedef enum psa_jpake_io_mode { - PSA_JPAKE_INPUT = 0, - PSA_JPAKE_OUTPUT = 1 -} psa_jpake_io_mode_t; - -struct psa_jpake_computation_stage_s { - /* The J-PAKE round we are currently on */ - psa_jpake_round_t MBEDTLS_PRIVATE(round); - /* The 'mode' we are currently in (inputting or outputting) */ - psa_jpake_io_mode_t MBEDTLS_PRIVATE(io_mode); - /* The number of completed inputs so far this round */ - uint8_t MBEDTLS_PRIVATE(inputs); - /* The number of completed outputs so far this round */ - uint8_t MBEDTLS_PRIVATE(outputs); - /* The next expected step (KEY_SHARE, ZK_PUBLIC or ZK_PROOF) */ - psa_pake_step_t MBEDTLS_PRIVATE(step); -}; - -#define PSA_JPAKE_EXPECTED_INPUTS(round) ((round) == PSA_JPAKE_FINISHED ? 0 : \ - ((round) == PSA_JPAKE_FIRST ? 2 : 1)) -#define PSA_JPAKE_EXPECTED_OUTPUTS(round) ((round) == PSA_JPAKE_FINISHED ? 0 : \ - ((round) == PSA_JPAKE_FIRST ? 2 : 1)) - -struct psa_pake_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_crypto_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - /* Algorithm of the PAKE operation */ - psa_algorithm_t MBEDTLS_PRIVATE(alg); - /* A primitive of type compatible with algorithm */ - psa_pake_primitive_t MBEDTLS_PRIVATE(primitive); - /* Stage of the PAKE operation: waiting for the setup, collecting inputs - * or computing. */ - uint8_t MBEDTLS_PRIVATE(stage); - /* Holds computation stage of the PAKE algorithms. */ - union { - uint8_t MBEDTLS_PRIVATE(dummy); -#if defined(PSA_WANT_ALG_JPAKE) - struct psa_jpake_computation_stage_s MBEDTLS_PRIVATE(jpake); -#endif - } MBEDTLS_PRIVATE(computation_stage); - union { - psa_driver_pake_context_t MBEDTLS_PRIVATE(ctx); - struct psa_crypto_driver_pake_inputs_s MBEDTLS_PRIVATE(inputs); - } MBEDTLS_PRIVATE(data); -#endif -}; - -/** \addtogroup pake - * @{ - */ - -/** The type of the data structure for PAKE cipher suites. - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. - */ -typedef struct psa_pake_cipher_suite_s psa_pake_cipher_suite_t; - -/** Return an initial value for a PAKE cipher suite object. - */ -static psa_pake_cipher_suite_t psa_pake_cipher_suite_init(void); - -/** Retrieve the PAKE algorithm from a PAKE cipher suite. - * - * \param[in] cipher_suite The cipher suite structure to query. - * - * \return The PAKE algorithm stored in the cipher suite structure. - */ -static psa_algorithm_t psa_pake_cs_get_algorithm( - const psa_pake_cipher_suite_t *cipher_suite); - -/** Declare the PAKE algorithm for the cipher suite. - * - * This function overwrites any PAKE algorithm - * previously set in \p cipher_suite. - * - * \note For #PSA_ALG_JPAKE, the only supported hash algorithm is SHA-256. - * - * \param[out] cipher_suite The cipher suite structure to write to. - * \param algorithm The PAKE algorithm to write. - * (`PSA_ALG_XXX` values of type ::psa_algorithm_t - * such that #PSA_ALG_IS_PAKE(\c alg) is true.) - * If this is 0, the PAKE algorithm in - * \p cipher_suite becomes unspecified. - */ -static void psa_pake_cs_set_algorithm(psa_pake_cipher_suite_t *cipher_suite, - psa_algorithm_t algorithm); - -/** Retrieve the primitive from a PAKE cipher suite. - * - * \param[in] cipher_suite The cipher suite structure to query. - * - * \return The primitive stored in the cipher suite structure. - */ -static psa_pake_primitive_t psa_pake_cs_get_primitive( - const psa_pake_cipher_suite_t *cipher_suite); - -/** Declare the primitive for a PAKE cipher suite. - * - * This function overwrites any primitive previously set in \p cipher_suite. - * - * \note For #PSA_ALG_JPAKE, the only supported primitive is ECC on the curve - * secp256r1, i.e. `PSA_PAKE_PRIMITIVE(PSA_PAKE_PRIMITIVE_TYPE_ECC, - * PSA_ECC_FAMILY_SECP_R1, 256)`. - * - * \param[out] cipher_suite The cipher suite structure to write to. - * \param primitive The primitive to write. If this is 0, the - * primitive type in \p cipher_suite becomes - * unspecified. - */ -static void psa_pake_cs_set_primitive(psa_pake_cipher_suite_t *cipher_suite, - psa_pake_primitive_t primitive); - -/** Retrieve the PAKE family from a PAKE cipher suite. - * - * \param[in] cipher_suite The cipher suite structure to query. - * - * \return The PAKE family stored in the cipher suite structure. - */ -static psa_pake_family_t psa_pake_cs_get_family( - const psa_pake_cipher_suite_t *cipher_suite); - -/** Retrieve the PAKE primitive bit-size from a PAKE cipher suite. - * - * \param[in] cipher_suite The cipher suite structure to query. - * - * \return The PAKE primitive bit-size stored in the cipher suite structure. - */ -static uint16_t psa_pake_cs_get_bits( - const psa_pake_cipher_suite_t *cipher_suite); - -/** Retrieve the hash algorithm from a PAKE cipher suite. - * - * \param[in] cipher_suite The cipher suite structure to query. - * - * \return The hash algorithm stored in the cipher suite structure. The return - * value is 0 if the PAKE is not parametrised by a hash algorithm or if - * the hash algorithm is not set. - */ -static psa_algorithm_t psa_pake_cs_get_hash( - const psa_pake_cipher_suite_t *cipher_suite); - -/** Declare the hash algorithm for a PAKE cipher suite. - * - * This function overwrites any hash algorithm - * previously set in \p cipher_suite. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * \param[out] cipher_suite The cipher suite structure to write to. - * \param hash The hash involved in the cipher suite. - * (`PSA_ALG_XXX` values of type ::psa_algorithm_t - * such that #PSA_ALG_IS_HASH(\c alg) is true.) - * If this is 0, the hash algorithm in - * \p cipher_suite becomes unspecified. - */ -static void psa_pake_cs_set_hash(psa_pake_cipher_suite_t *cipher_suite, - psa_algorithm_t hash); - -/** The type of the state data structure for PAKE operations. - * - * Before calling any function on a PAKE operation object, the application - * must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_pake_operation_t operation; - * memset(&operation, 0, sizeof(operation)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_pake_operation_t operation = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_PAKE_OPERATION_INIT, - * for example: - * \code - * psa_pake_operation_t operation = PSA_PAKE_OPERATION_INIT; - * \endcode - * - Assign the result of the function psa_pake_operation_init() - * to the structure, for example: - * \code - * psa_pake_operation_t operation; - * operation = psa_pake_operation_init(); - * \endcode - * - * This is an implementation-defined \c struct. Applications should not - * make any assumptions about the content of this structure. - * Implementation details can change in future versions without notice. */ -typedef struct psa_pake_operation_s psa_pake_operation_t; - -/** The type of input values for PAKE operations. */ -typedef struct psa_crypto_driver_pake_inputs_s psa_crypto_driver_pake_inputs_t; - -/** The type of computation stage for J-PAKE operations. */ -typedef struct psa_jpake_computation_stage_s psa_jpake_computation_stage_t; - -/** Return an initial value for a PAKE operation object. - */ -static psa_pake_operation_t psa_pake_operation_init(void); - -/** Get the length of the password in bytes from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] password_len Password length. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * Password hasn't been set yet. - */ -psa_status_t psa_crypto_driver_pake_get_password_len( - const psa_crypto_driver_pake_inputs_t *inputs, - size_t *password_len); - -/** Get the password from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] buffer Return buffer for password. - * \param buffer_size Size of the return buffer in bytes. - * \param[out] buffer_length Actual size of the password in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * Password hasn't been set yet. - */ -psa_status_t psa_crypto_driver_pake_get_password( - const psa_crypto_driver_pake_inputs_t *inputs, - uint8_t *buffer, size_t buffer_size, size_t *buffer_length); - -/** Get the length of the user id in bytes from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] user_len User id length. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * User id hasn't been set yet. - */ -psa_status_t psa_crypto_driver_pake_get_user_len( - const psa_crypto_driver_pake_inputs_t *inputs, - size_t *user_len); - -/** Get the length of the peer id in bytes from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] peer_len Peer id length. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * Peer id hasn't been set yet. - */ -psa_status_t psa_crypto_driver_pake_get_peer_len( - const psa_crypto_driver_pake_inputs_t *inputs, - size_t *peer_len); - -/** Get the user id from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] user_id User id. - * \param user_id_size Size of \p user_id in bytes. - * \param[out] user_id_len Size of the user id in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * User id hasn't been set yet. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p user_id is too small. - */ -psa_status_t psa_crypto_driver_pake_get_user( - const psa_crypto_driver_pake_inputs_t *inputs, - uint8_t *user_id, size_t user_id_size, size_t *user_id_len); - -/** Get the peer id from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] peer_id Peer id. - * \param peer_id_size Size of \p peer_id in bytes. - * \param[out] peer_id_length Size of the peer id in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * Peer id hasn't been set yet. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p peer_id is too small. - */ -psa_status_t psa_crypto_driver_pake_get_peer( - const psa_crypto_driver_pake_inputs_t *inputs, - uint8_t *peer_id, size_t peer_id_size, size_t *peer_id_length); - -/** Get the cipher suite from given inputs. - * - * \param[in] inputs Operation inputs. - * \param[out] cipher_suite Return buffer for role. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BAD_STATE - * Cipher_suite hasn't been set yet. - */ -psa_status_t psa_crypto_driver_pake_get_cipher_suite( - const psa_crypto_driver_pake_inputs_t *inputs, - psa_pake_cipher_suite_t *cipher_suite); - -/** Set the session information for a password-authenticated key exchange. - * - * The sequence of operations to set up a password-authenticated key exchange - * is as follows: - * -# Allocate an operation object which will be passed to all the functions - * listed here. - * -# Initialize the operation object with one of the methods described in the - * documentation for #psa_pake_operation_t, e.g. - * #PSA_PAKE_OPERATION_INIT. - * -# Call psa_pake_setup() to specify the cipher suite. - * -# Call \c psa_pake_set_xxx() functions on the operation to complete the - * setup. The exact sequence of \c psa_pake_set_xxx() functions that needs - * to be called depends on the algorithm in use. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * A typical sequence of calls to perform a password-authenticated key - * exchange: - * -# Call psa_pake_output(operation, #PSA_PAKE_STEP_KEY_SHARE, ...) to get the - * key share that needs to be sent to the peer. - * -# Call psa_pake_input(operation, #PSA_PAKE_STEP_KEY_SHARE, ...) to provide - * the key share that was received from the peer. - * -# Depending on the algorithm additional calls to psa_pake_output() and - * psa_pake_input() might be necessary. - * -# Call psa_pake_get_implicit_key() for accessing the shared secret. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * If an error occurs at any step after a call to psa_pake_setup(), - * the operation will need to be reset by a call to psa_pake_abort(). The - * application may call psa_pake_abort() at any time after the operation - * has been initialized. - * - * After a successful call to psa_pake_setup(), the application must - * eventually terminate the operation. The following events terminate an - * operation: - * - A call to psa_pake_abort(). - * - A successful call to psa_pake_get_implicit_key(). - * - * \param[in,out] operation The operation object to set up. It must have - * been initialized but not set up yet. - * \param[in] cipher_suite The cipher suite to use. (A cipher suite fully - * characterizes a PAKE algorithm and determines - * the algorithm as well.) - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The algorithm in \p cipher_suite is not a PAKE algorithm, or the - * PAKE primitive in \p cipher_suite is not compatible with the - * PAKE algorithm, or the hash algorithm in \p cipher_suite is invalid - * or not compatible with the PAKE algorithm and primitive. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The algorithm in \p cipher_suite is not a supported PAKE algorithm, - * or the PAKE primitive in \p cipher_suite is not supported or not - * compatible with the PAKE algorithm, or the hash algorithm in - * \p cipher_suite is not supported or not compatible with the PAKE - * algorithm and primitive. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_setup(psa_pake_operation_t *operation, - const psa_pake_cipher_suite_t *cipher_suite); - -/** Set the password for a password-authenticated key exchange from key ID. - * - * Call this function when the password, or a value derived from the password, - * is already present in the key store. - * - * \param[in,out] operation The operation object to set the password for. It - * must have been set up by psa_pake_setup() and - * not yet in use (neither psa_pake_output() nor - * psa_pake_input() has been called yet). It must - * be on operation for which the password hasn't - * been set yet (psa_pake_set_password_key() - * hasn't been called yet). - * \param password Identifier of the key holding the password or a - * value derived from the password (eg. by a - * memory-hard function). It must remain valid - * until the operation terminates. It must be of - * type #PSA_KEY_TYPE_PASSWORD or - * #PSA_KEY_TYPE_PASSWORD_HASH. It has to allow - * the usage #PSA_KEY_USAGE_DERIVE. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_HANDLE - * \p password is not a valid key identifier. - * \retval #PSA_ERROR_NOT_PERMITTED - * The key does not have the #PSA_KEY_USAGE_DERIVE flag, or it does not - * permit the \p operation's algorithm. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The key type for \p password is not #PSA_KEY_TYPE_PASSWORD or - * #PSA_KEY_TYPE_PASSWORD_HASH, or \p password is not compatible with - * the \p operation's cipher suite. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The key type or key size of \p password is not supported with the - * \p operation's cipher suite. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must have been set up.), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_set_password_key(psa_pake_operation_t *operation, - mbedtls_svc_key_id_t password); - -/** Set the user ID for a password-authenticated key exchange. - * - * Call this function to set the user ID. For PAKE algorithms that associate a - * user identifier with each side of the session you need to call - * psa_pake_set_peer() as well. For PAKE algorithms that associate a single - * user identifier with the session, call psa_pake_set_user() only. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * \note When using the built-in implementation of #PSA_ALG_JPAKE, the user ID - * must be `"client"` (6-byte string) or `"server"` (6-byte string). - * Third-party drivers may or may not have this limitation. - * - * \param[in,out] operation The operation object to set the user ID for. It - * must have been set up by psa_pake_setup() and - * not yet in use (neither psa_pake_output() nor - * psa_pake_input() has been called yet). It must - * be on operation for which the user ID hasn't - * been set (psa_pake_set_user() hasn't been - * called yet). - * \param[in] user_id The user ID to authenticate with. - * \param user_id_len Size of the \p user_id buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p user_id is not valid for the \p operation's algorithm and cipher - * suite. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The value of \p user_id is not supported by the implementation. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_set_user(psa_pake_operation_t *operation, - const uint8_t *user_id, - size_t user_id_len); - -/** Set the peer ID for a password-authenticated key exchange. - * - * Call this function in addition to psa_pake_set_user() for PAKE algorithms - * that associate a user identifier with each side of the session. For PAKE - * algorithms that associate a single user identifier with the session, call - * psa_pake_set_user() only. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * \note When using the built-in implementation of #PSA_ALG_JPAKE, the peer ID - * must be `"client"` (6-byte string) or `"server"` (6-byte string). - * Third-party drivers may or may not have this limitation. - * - * \param[in,out] operation The operation object to set the peer ID for. It - * must have been set up by psa_pake_setup() and - * not yet in use (neither psa_pake_output() nor - * psa_pake_input() has been called yet). It must - * be on operation for which the peer ID hasn't - * been set (psa_pake_set_peer() hasn't been - * called yet). - * \param[in] peer_id The peer's ID to authenticate. - * \param peer_id_len Size of the \p peer_id buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p peer_id is not valid for the \p operation's algorithm and cipher - * suite. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The algorithm doesn't associate a second identity with the session. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * Calling psa_pake_set_peer() is invalid with the \p operation's - * algorithm, the operation state is not valid, or the library has not - * been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_set_peer(psa_pake_operation_t *operation, - const uint8_t *peer_id, - size_t peer_id_len); - -/** Set the application role for a password-authenticated key exchange. - * - * Not all PAKE algorithms need to differentiate the communicating entities. - * It is optional to call this function for PAKEs that don't require a role - * to be specified. For such PAKEs the application role parameter is ignored, - * or #PSA_PAKE_ROLE_NONE can be passed as \c role. - * - * Refer to the documentation of individual PAKE algorithm types (`PSA_ALG_XXX` - * values of type ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) - * for more information. - * - * \param[in,out] operation The operation object to specify the - * application's role for. It must have been set up - * by psa_pake_setup() and not yet in use (neither - * psa_pake_output() nor psa_pake_input() has been - * called yet). It must be on operation for which - * the application's role hasn't been specified - * (psa_pake_set_role() hasn't been called yet). - * \param role A value of type ::psa_pake_role_t indicating the - * application's role in the PAKE the algorithm - * that is being set up. For more information see - * the documentation of \c PSA_PAKE_ROLE_XXX - * constants. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The \p role is not a valid PAKE role in the \p operation’s algorithm. - * \retval #PSA_ERROR_NOT_SUPPORTED - * The \p role for this algorithm is not supported or is not valid. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid, or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_set_role(psa_pake_operation_t *operation, - psa_pake_role_t role); - -/** Get output for a step of a password-authenticated key exchange. - * - * Depending on the algorithm being executed, you might need to call this - * function several times or you might not need to call this at all. - * - * The exact sequence of calls to perform a password-authenticated key - * exchange depends on the algorithm in use. Refer to the documentation of - * individual PAKE algorithm types (`PSA_ALG_XXX` values of type - * ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) for more - * information. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_pake_abort(). - * - * \param[in,out] operation Active PAKE operation. - * \param step The step of the algorithm for which the output is - * requested. - * \param[out] output Buffer where the output is to be written in the - * format appropriate for this \p step. Refer to - * the documentation of the individual - * \c PSA_PAKE_STEP_XXX constants for more - * information. - * \param output_size Size of the \p output buffer in bytes. This must - * be at least #PSA_PAKE_OUTPUT_SIZE(\c alg, \c - * primitive, \p output_step) where \c alg and - * \p primitive are the PAKE algorithm and primitive - * in the operation's cipher suite, and \p step is - * the output step. - * - * \param[out] output_length On success, the number of bytes of the returned - * output. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_BUFFER_TOO_SMALL - * The size of the \p output buffer is too small. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p step is not compatible with the operation's algorithm. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p step is not supported with the operation's algorithm. - * \retval #PSA_ERROR_INSUFFICIENT_ENTROPY \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, and fully set - * up, and this call must conform to the algorithm's requirements - * for ordering of input and output steps), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_output(psa_pake_operation_t *operation, - psa_pake_step_t step, - uint8_t *output, - size_t output_size, - size_t *output_length); - -/** Provide input for a step of a password-authenticated key exchange. - * - * Depending on the algorithm being executed, you might need to call this - * function several times or you might not need to call this at all. - * - * The exact sequence of calls to perform a password-authenticated key - * exchange depends on the algorithm in use. Refer to the documentation of - * individual PAKE algorithm types (`PSA_ALG_XXX` values of type - * ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) for more - * information. - * - * If this function returns an error status, the operation enters an error - * state and must be aborted by calling psa_pake_abort(). - * - * \param[in,out] operation Active PAKE operation. - * \param step The step for which the input is provided. - * \param[in] input Buffer containing the input in the format - * appropriate for this \p step. Refer to the - * documentation of the individual - * \c PSA_PAKE_STEP_XXX constants for more - * information. - * \param input_length Size of the \p input buffer in bytes. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The verification fails for a #PSA_PAKE_STEP_ZK_PROOF input step. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * \p input_length is not compatible with the \p operation’s algorithm, - * or the \p input is not valid for the \p operation's algorithm, - * cipher suite or \p step. - * \retval #PSA_ERROR_NOT_SUPPORTED - * \p step p is not supported with the \p operation's algorithm, or the - * \p input is not supported for the \p operation's algorithm, cipher - * suite or \p step. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The operation state is not valid (it must be active, and fully set - * up, and this call must conform to the algorithm's requirements - * for ordering of input and output steps), or - * the library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_input(psa_pake_operation_t *operation, - psa_pake_step_t step, - const uint8_t *input, - size_t input_length); - -/** Get implicitly confirmed shared secret from a PAKE. - * - * At this point there is a cryptographic guarantee that only the authenticated - * party who used the same password is able to compute the key. But there is no - * guarantee that the peer is the party it claims to be and was able to do so. - * - * That is, the authentication is only implicit. Since the peer is not - * authenticated yet, no action should be taken yet that assumes that the peer - * is who it claims to be. For example, do not access restricted files on the - * peer's behalf until an explicit authentication has succeeded. - * - * This function can be called after the key exchange phase of the operation - * has completed. It imports the shared secret output of the PAKE into the - * provided derivation operation. The input step - * #PSA_KEY_DERIVATION_INPUT_SECRET is used when placing the shared key - * material in the key derivation operation. - * - * The exact sequence of calls to perform a password-authenticated key - * exchange depends on the algorithm in use. Refer to the documentation of - * individual PAKE algorithm types (`PSA_ALG_XXX` values of type - * ::psa_algorithm_t such that #PSA_ALG_IS_PAKE(\c alg) is true) for more - * information. - * - * When this function returns successfully, \p operation becomes inactive. - * If this function returns an error status, both \p operation - * and \c key_derivation operations enter an error state and must be aborted by - * calling psa_pake_abort() and psa_key_derivation_abort() respectively. - * - * \param[in,out] operation Active PAKE operation. - * \param[out] output A key derivation operation that is ready - * for an input step of type - * #PSA_KEY_DERIVATION_INPUT_SECRET. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * #PSA_KEY_DERIVATION_INPUT_SECRET is not compatible with the - * algorithm in the \p output key derivation operation. - * \retval #PSA_ERROR_NOT_SUPPORTED - * Input from a PAKE is not supported by the algorithm in the \p output - * key derivation operation. - * \retval #PSA_ERROR_INSUFFICIENT_MEMORY \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_STORAGE_FAILURE \emptydescription - * \retval #PSA_ERROR_DATA_CORRUPT \emptydescription - * \retval #PSA_ERROR_DATA_INVALID \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The PAKE operation state is not valid (it must be active, but beyond - * that validity is specific to the algorithm), or - * the library has not been previously initialized by psa_crypto_init(), - * or the state of \p output is not valid for - * the #PSA_KEY_DERIVATION_INPUT_SECRET step. This can happen if the - * step is out of order or the application has done this step already - * and it may not be repeated. - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_get_implicit_key(psa_pake_operation_t *operation, - psa_key_derivation_operation_t *output); - -/** Abort a PAKE operation. - * - * Aborting an operation frees all associated resources except for the \c - * operation structure itself. Once aborted, the operation object can be reused - * for another operation by calling psa_pake_setup() again. - * - * This function may be called at any time after the operation - * object has been initialized as described in #psa_pake_operation_t. - * - * In particular, calling psa_pake_abort() after the operation has been - * terminated by a call to psa_pake_abort() or psa_pake_get_implicit_key() - * is safe and has no effect. - * - * \param[in,out] operation The operation to abort. - * - * \retval #PSA_SUCCESS - * Success. - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - * \retval #PSA_ERROR_BAD_STATE - * The library has not been previously initialized by psa_crypto_init(). - * It is implementation-dependent whether a failure to initialize - * results in this error code. - */ -psa_status_t psa_pake_abort(psa_pake_operation_t *operation); - -/**@}*/ - -static inline psa_algorithm_t psa_pake_cs_get_algorithm( - const psa_pake_cipher_suite_t *cipher_suite) -{ - return cipher_suite->algorithm; -} - -static inline void psa_pake_cs_set_algorithm( - psa_pake_cipher_suite_t *cipher_suite, - psa_algorithm_t algorithm) -{ - if (!PSA_ALG_IS_PAKE(algorithm)) { - cipher_suite->algorithm = 0; - } else { - cipher_suite->algorithm = algorithm; - } -} - -static inline psa_pake_primitive_t psa_pake_cs_get_primitive( - const psa_pake_cipher_suite_t *cipher_suite) -{ - return PSA_PAKE_PRIMITIVE(cipher_suite->type, cipher_suite->family, - cipher_suite->bits); -} - -static inline void psa_pake_cs_set_primitive( - psa_pake_cipher_suite_t *cipher_suite, - psa_pake_primitive_t primitive) -{ - cipher_suite->type = (psa_pake_primitive_type_t) (primitive >> 24); - cipher_suite->family = (psa_pake_family_t) (0xFF & (primitive >> 16)); - cipher_suite->bits = (uint16_t) (0xFFFF & primitive); -} - -static inline psa_pake_family_t psa_pake_cs_get_family( - const psa_pake_cipher_suite_t *cipher_suite) -{ - return cipher_suite->family; -} - -static inline uint16_t psa_pake_cs_get_bits( - const psa_pake_cipher_suite_t *cipher_suite) -{ - return cipher_suite->bits; -} - -static inline psa_algorithm_t psa_pake_cs_get_hash( - const psa_pake_cipher_suite_t *cipher_suite) -{ - return cipher_suite->hash; -} - -static inline void psa_pake_cs_set_hash(psa_pake_cipher_suite_t *cipher_suite, - psa_algorithm_t hash) -{ - if (!PSA_ALG_IS_HASH(hash)) { - cipher_suite->hash = 0; - } else { - cipher_suite->hash = hash; - } -} - -static inline struct psa_pake_cipher_suite_s psa_pake_cipher_suite_init(void) -{ - const struct psa_pake_cipher_suite_s v = PSA_PAKE_CIPHER_SUITE_INIT; - return v; -} - -static inline struct psa_pake_operation_s psa_pake_operation_init(void) -{ - const struct psa_pake_operation_s v = PSA_PAKE_OPERATION_INIT; - return v; -} - -#ifdef __cplusplus -} -#endif - -#endif /* PSA_CRYPTO_EXTRA_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_legacy.h b/android/mbedtls/arm64-v8a/include/psa/crypto_legacy.h deleted file mode 100644 index 7df3614d6..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_legacy.h +++ /dev/null @@ -1,88 +0,0 @@ -/** - * \file psa/crypto_legacy.h - * - * \brief Add temporary suppport for deprecated symbols before they are - * removed from the library. - * - * PSA_WANT_KEY_TYPE_xxx_KEY_PAIR and MBEDTLS_PSA_ACCEL_KEY_TYPE_xxx_KEY_PAIR - * symbols are deprecated. - * New symols add a suffix to that base name in order to clearly state what is - * the expected use for the key (use, import, export, generate, derive). - * Here we define some backward compatibility support for uses stil using - * the legacy symbols. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef MBEDTLS_PSA_CRYPTO_LEGACY_H -#define MBEDTLS_PSA_CRYPTO_LEGACY_H - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR) //no-check-names -#if !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_GENERATE 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#define PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_DERIVE 1 -#endif -#endif - -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR) //no-check-names -#if !defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT 1 -#endif -#if !defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE) -#define PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_GENERATE 1 -#endif -#endif - -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR) //no-check-names -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_BASIC -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_IMPORT -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_EXPORT -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_GENERATE) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_GENERATE -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_ECC_KEY_PAIR_DERIVE -#endif -#endif - -#if defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR) //no-check-names -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_BASIC) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_BASIC -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_IMPORT) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_IMPORT -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_EXPORT) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_EXPORT -#endif -#if !defined(MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_GENERATE) -#define MBEDTLS_PSA_ACCEL_KEY_TYPE_RSA_KEY_PAIR_GENERATE -#endif -#endif - -#endif /* MBEDTLS_PSA_CRYPTO_LEGACY_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_platform.h b/android/mbedtls/arm64-v8a/include/psa/crypto_platform.h deleted file mode 100644 index a871ee124..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_platform.h +++ /dev/null @@ -1,102 +0,0 @@ -/** - * \file psa/crypto_platform.h - * - * \brief PSA cryptography module: Mbed TLS platform definitions - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * This file contains platform-dependent type definitions. - * - * In implementations with isolation between the application and the - * cryptography module, implementers should take care to ensure that - * the definitions that are exposed to applications match what the - * module implements. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_PLATFORM_H -#define PSA_CRYPTO_PLATFORM_H -#include "mbedtls/private_access.h" - -/* - * Include the build-time configuration information header. Here, we do not - * include `"mbedtls/build_info.h"` directly but `"psa/build_info.h"`, which - * is basically just an alias to it. This is to ease the maintenance of the - * TF-PSA-Crypto repository which has a different build system and - * configuration. - */ -#include "psa/build_info.h" - -/* PSA requires several types which C99 provides in stdint.h. */ -#include - -#if defined(MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER) - -/* Building for the PSA Crypto service on a PSA platform, a key owner is a PSA - * partition identifier. - * - * The function psa_its_identifier_of_slot() in psa_crypto_storage.c that - * translates a key identifier to a key storage file name assumes that - * mbedtls_key_owner_id_t is a 32-bit integer. This function thus needs - * reworking if mbedtls_key_owner_id_t is not defined as a 32-bit integer - * here anymore. - */ -typedef int32_t mbedtls_key_owner_id_t; - -/** Compare two key owner identifiers. - * - * \param id1 First key owner identifier. - * \param id2 Second key owner identifier. - * - * \return Non-zero if the two key owner identifiers are equal, zero otherwise. - */ -static inline int mbedtls_key_owner_id_equal(mbedtls_key_owner_id_t id1, - mbedtls_key_owner_id_t id2) -{ - return id1 == id2; -} - -#endif /* MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER */ - -/* - * When MBEDTLS_PSA_CRYPTO_SPM is defined, the code is being built for SPM - * (Secure Partition Manager) integration which separates the code into two - * parts: NSPE (Non-Secure Processing Environment) and SPE (Secure Processing - * Environment). When building for the SPE, an additional header file should be - * included. - */ -#if defined(MBEDTLS_PSA_CRYPTO_SPM) -#define PSA_CRYPTO_SECURE 1 -#include "crypto_spe.h" -#endif // MBEDTLS_PSA_CRYPTO_SPM - -#if defined(MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG) -/** The type of the context passed to mbedtls_psa_external_get_random(). - * - * Mbed TLS initializes the context to all-bits-zero before calling - * mbedtls_psa_external_get_random() for the first time. - * - * The definition of this type in the Mbed TLS source code is for - * demonstration purposes. Implementers of mbedtls_psa_external_get_random() - * are expected to replace it with a custom definition. - */ -typedef struct { - uintptr_t MBEDTLS_PRIVATE(opaque)[2]; -} mbedtls_psa_external_random_context_t; -#endif /* MBEDTLS_PSA_CRYPTO_EXTERNAL_RNG */ - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -/** The type of the client handle used in context structures - * - * When a client view of the multipart context structures is required, - * this handle is used to keep a mapping with the service side of the - * context which contains the actual data. - */ -typedef uint32_t mbedtls_psa_client_handle_t; -#endif - -#endif /* PSA_CRYPTO_PLATFORM_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_se_driver.h b/android/mbedtls/arm64-v8a/include/psa/crypto_se_driver.h deleted file mode 100644 index 9ce14bba6..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_se_driver.h +++ /dev/null @@ -1,1383 +0,0 @@ -/** - * \file psa/crypto_se_driver.h - * \brief PSA external cryptoprocessor driver module - * - * This header declares types and function signatures for cryptography - * drivers that access key material via opaque references. - * This is meant for cryptoprocessors that have a separate key storage from the - * space in which the PSA Crypto implementation runs, typically secure - * elements (SEs). - * - * This file is part of the PSA Crypto Driver HAL (hardware abstraction layer), - * containing functions for driver developers to implement to enable hardware - * to be called in a standardized way by a PSA Cryptography API - * implementation. The functions comprising the driver HAL, which driver - * authors implement, are not intended to be called by application developers. - */ - -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ -#ifndef PSA_CRYPTO_SE_DRIVER_H -#define PSA_CRYPTO_SE_DRIVER_H -#include "mbedtls/private_access.h" - -#include "crypto_driver_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** \defgroup se_init Secure element driver initialization - */ -/**@{*/ - -/** \brief Driver context structure - * - * Driver functions receive a pointer to this structure. - * Each registered driver has one instance of this structure. - * - * Implementations must include the fields specified here and - * may include other fields. - */ -typedef struct { - /** A read-only pointer to the driver's persistent data. - * - * Drivers typically use this persistent data to keep track of - * which slot numbers are available. This is only a guideline: - * drivers may use the persistent data for any purpose, keeping - * in mind the restrictions on when the persistent data is saved - * to storage: the persistent data is only saved after calling - * certain functions that receive a writable pointer to the - * persistent data. - * - * The core allocates a memory buffer for the persistent data. - * The pointer is guaranteed to be suitably aligned for any data type, - * like a pointer returned by `malloc` (but the core can use any - * method to allocate the buffer, not necessarily `malloc`). - * - * The size of this buffer is in the \c persistent_data_size field of - * this structure. - * - * Before the driver is initialized for the first time, the content of - * the persistent data is all-bits-zero. After a driver upgrade, if the - * size of the persistent data has increased, the original data is padded - * on the right with zeros; if the size has decreased, the original data - * is truncated to the new size. - * - * This pointer is to read-only data. Only a few driver functions are - * allowed to modify the persistent data. These functions receive a - * writable pointer. These functions are: - * - psa_drv_se_t::p_init - * - psa_drv_se_key_management_t::p_allocate - * - psa_drv_se_key_management_t::p_destroy - * - * The PSA Cryptography core saves the persistent data from one - * session to the next. It does this before returning from API functions - * that call a driver method that is allowed to modify the persistent - * data, specifically: - * - psa_crypto_init() causes a call to psa_drv_se_t::p_init, and may call - * psa_drv_se_key_management_t::p_destroy to complete an action - * that was interrupted by a power failure. - * - Key creation functions cause a call to - * psa_drv_se_key_management_t::p_allocate, and may cause a call to - * psa_drv_se_key_management_t::p_destroy in case an error occurs. - * - psa_destroy_key() causes a call to - * psa_drv_se_key_management_t::p_destroy. - */ - const void *const MBEDTLS_PRIVATE(persistent_data); - - /** The size of \c persistent_data in bytes. - * - * This is always equal to the value of the `persistent_data_size` field - * of the ::psa_drv_se_t structure when the driver is registered. - */ - const size_t MBEDTLS_PRIVATE(persistent_data_size); - - /** Driver transient data. - * - * The core initializes this value to 0 and does not read or modify it - * afterwards. The driver may store whatever it wants in this field. - */ - uintptr_t MBEDTLS_PRIVATE(transient_data); -} psa_drv_se_context_t; - -/** \brief A driver initialization function. - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] persistent_data A pointer to the persistent data - * that allows writing. - * \param location The location value for which this driver - * is registered. The driver will be invoked - * for all keys whose lifetime is in this - * location. - * - * \retval #PSA_SUCCESS - * The driver is operational. - * The core will update the persistent data in storage. - * \return - * Any other return value prevents the driver from being used in - * this session. - * The core will NOT update the persistent data in storage. - */ -typedef psa_status_t (*psa_drv_se_init_t)(psa_drv_se_context_t *drv_context, - void *persistent_data, - psa_key_location_t location); - -#if defined(__DOXYGEN_ONLY__) || !defined(MBEDTLS_PSA_CRYPTO_SE_C) -/* Mbed TLS with secure element support enabled defines this type in - * crypto_types.h because it is also visible to applications through an - * implementation-specific extension. - * For the PSA Cryptography specification, this type is only visible - * via crypto_se_driver.h. */ -/** An internal designation of a key slot between the core part of the - * PSA Crypto implementation and the driver. The meaning of this value - * is driver-dependent. */ -typedef uint64_t psa_key_slot_number_t; -#endif /* __DOXYGEN_ONLY__ || !MBEDTLS_PSA_CRYPTO_SE_C */ - -/**@}*/ - -/** \defgroup se_mac Secure Element Message Authentication Codes - * Generation and authentication of Message Authentication Codes (MACs) using - * a secure element can be done either as a single function call (via the - * `psa_drv_se_mac_generate_t` or `psa_drv_se_mac_verify_t` functions), or in - * parts using the following sequence: - * - `psa_drv_se_mac_setup_t` - * - `psa_drv_se_mac_update_t` - * - `psa_drv_se_mac_update_t` - * - ... - * - `psa_drv_se_mac_finish_t` or `psa_drv_se_mac_finish_verify_t` - * - * If a previously started secure element MAC operation needs to be terminated, - * it should be done so by the `psa_drv_se_mac_abort_t`. Failure to do so may - * result in allocated resources not being freed or in other undefined - * behavior. - */ -/**@{*/ -/** \brief A function that starts a secure element MAC operation for a PSA - * Crypto Driver implementation - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] op_context A structure that will contain the - * hardware-specific MAC context - * \param[in] key_slot The slot of the key to be used for the - * operation - * \param[in] algorithm The algorithm to be used to underly the MAC - * operation - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_mac_setup_t)(psa_drv_se_context_t *drv_context, - void *op_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t algorithm); - -/** \brief A function that continues a previously started secure element MAC - * operation - * - * \param[in,out] op_context A hardware-specific structure for the - * previously-established MAC operation to be - * updated - * \param[in] p_input A buffer containing the message to be appended - * to the MAC operation - * \param[in] input_length The size in bytes of the input message buffer - */ -typedef psa_status_t (*psa_drv_se_mac_update_t)(void *op_context, - const uint8_t *p_input, - size_t input_length); - -/** \brief a function that completes a previously started secure element MAC - * operation by returning the resulting MAC. - * - * \param[in,out] op_context A hardware-specific structure for the - * previously started MAC operation to be - * finished - * \param[out] p_mac A buffer where the generated MAC will be - * placed - * \param[in] mac_size The size in bytes of the buffer that has been - * allocated for the `output` buffer - * \param[out] p_mac_length After completion, will contain the number of - * bytes placed in the `p_mac` buffer - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_mac_finish_t)(void *op_context, - uint8_t *p_mac, - size_t mac_size, - size_t *p_mac_length); - -/** \brief A function that completes a previously started secure element MAC - * operation by comparing the resulting MAC against a provided value - * - * \param[in,out] op_context A hardware-specific structure for the previously - * started MAC operation to be finished - * \param[in] p_mac The MAC value against which the resulting MAC - * will be compared against - * \param[in] mac_length The size in bytes of the value stored in `p_mac` - * - * \retval #PSA_SUCCESS - * The operation completed successfully and the MACs matched each - * other - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The operation completed successfully, but the calculated MAC did - * not match the provided MAC - */ -typedef psa_status_t (*psa_drv_se_mac_finish_verify_t)(void *op_context, - const uint8_t *p_mac, - size_t mac_length); - -/** \brief A function that aborts a previous started secure element MAC - * operation - * - * \param[in,out] op_context A hardware-specific structure for the previously - * started MAC operation to be aborted - */ -typedef psa_status_t (*psa_drv_se_mac_abort_t)(void *op_context); - -/** \brief A function that performs a secure element MAC operation in one - * command and returns the calculated MAC - * - * \param[in,out] drv_context The driver context structure. - * \param[in] p_input A buffer containing the message to be MACed - * \param[in] input_length The size in bytes of `p_input` - * \param[in] key_slot The slot of the key to be used - * \param[in] alg The algorithm to be used to underlie the MAC - * operation - * \param[out] p_mac A buffer where the generated MAC will be - * placed - * \param[in] mac_size The size in bytes of the `p_mac` buffer - * \param[out] p_mac_length After completion, will contain the number of - * bytes placed in the `output` buffer - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_mac_generate_t)(psa_drv_se_context_t *drv_context, - const uint8_t *p_input, - size_t input_length, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - uint8_t *p_mac, - size_t mac_size, - size_t *p_mac_length); - -/** \brief A function that performs a secure element MAC operation in one - * command and compares the resulting MAC against a provided value - * - * \param[in,out] drv_context The driver context structure. - * \param[in] p_input A buffer containing the message to be MACed - * \param[in] input_length The size in bytes of `input` - * \param[in] key_slot The slot of the key to be used - * \param[in] alg The algorithm to be used to underlie the MAC - * operation - * \param[in] p_mac The MAC value against which the resulting MAC will - * be compared against - * \param[in] mac_length The size in bytes of `mac` - * - * \retval #PSA_SUCCESS - * The operation completed successfully and the MACs matched each - * other - * \retval #PSA_ERROR_INVALID_SIGNATURE - * The operation completed successfully, but the calculated MAC did - * not match the provided MAC - */ -typedef psa_status_t (*psa_drv_se_mac_verify_t)(psa_drv_se_context_t *drv_context, - const uint8_t *p_input, - size_t input_length, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - const uint8_t *p_mac, - size_t mac_length); - -/** \brief A struct containing all of the function pointers needed to - * perform secure element MAC operations - * - * PSA Crypto API implementations should populate the table as appropriate - * upon startup. - * - * If one of the functions is not implemented (such as - * `psa_drv_se_mac_generate_t`), it should be set to NULL. - * - * Driver implementers should ensure that they implement all of the functions - * that make sense for their hardware, and that they provide a full solution - * (for example, if they support `p_setup`, they should also support - * `p_update` and at least one of `p_finish` or `p_finish_verify`). - * - */ -typedef struct { - /**The size in bytes of the hardware-specific secure element MAC context - * structure - */ - size_t MBEDTLS_PRIVATE(context_size); - /** Function that performs a MAC setup operation - */ - psa_drv_se_mac_setup_t MBEDTLS_PRIVATE(p_setup); - /** Function that performs a MAC update operation - */ - psa_drv_se_mac_update_t MBEDTLS_PRIVATE(p_update); - /** Function that completes a MAC operation - */ - psa_drv_se_mac_finish_t MBEDTLS_PRIVATE(p_finish); - /** Function that completes a MAC operation with a verify check - */ - psa_drv_se_mac_finish_verify_t MBEDTLS_PRIVATE(p_finish_verify); - /** Function that aborts a previously started MAC operation - */ - psa_drv_se_mac_abort_t MBEDTLS_PRIVATE(p_abort); - /** Function that performs a MAC operation in one call - */ - psa_drv_se_mac_generate_t MBEDTLS_PRIVATE(p_mac); - /** Function that performs a MAC and verify operation in one call - */ - psa_drv_se_mac_verify_t MBEDTLS_PRIVATE(p_mac_verify); -} psa_drv_se_mac_t; -/**@}*/ - -/** \defgroup se_cipher Secure Element Symmetric Ciphers - * - * Encryption and Decryption using secure element keys in block modes other - * than ECB must be done in multiple parts, using the following flow: - * - `psa_drv_se_cipher_setup_t` - * - `psa_drv_se_cipher_set_iv_t` (optional depending upon block mode) - * - `psa_drv_se_cipher_update_t` - * - `psa_drv_se_cipher_update_t` - * - ... - * - `psa_drv_se_cipher_finish_t` - * - * If a previously started secure element Cipher operation needs to be - * terminated, it should be done so by the `psa_drv_se_cipher_abort_t`. Failure - * to do so may result in allocated resources not being freed or in other - * undefined behavior. - * - * In situations where a PSA Cryptographic API implementation is using a block - * mode not-supported by the underlying hardware or driver, it can construct - * the block mode itself, while calling the `psa_drv_se_cipher_ecb_t` function - * for the cipher operations. - */ -/**@{*/ - -/** \brief A function that provides the cipher setup function for a - * secure element driver - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] op_context A structure that will contain the - * hardware-specific cipher context. - * \param[in] key_slot The slot of the key to be used for the - * operation - * \param[in] algorithm The algorithm to be used in the cipher - * operation - * \param[in] direction Indicates whether the operation is an encrypt - * or decrypt - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - */ -typedef psa_status_t (*psa_drv_se_cipher_setup_t)(psa_drv_se_context_t *drv_context, - void *op_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t algorithm, - psa_encrypt_or_decrypt_t direction); - -/** \brief A function that sets the initialization vector (if - * necessary) for a secure element cipher operation - * - * Rationale: The `psa_se_cipher_*` operation in the PSA Cryptographic API has - * two IV functions: one to set the IV, and one to generate it internally. The - * generate function is not necessary for the drivers to implement as the PSA - * Crypto implementation can do the generation using its RNG features. - * - * \param[in,out] op_context A structure that contains the previously set up - * hardware-specific cipher context - * \param[in] p_iv A buffer containing the initialization vector - * \param[in] iv_length The size (in bytes) of the `p_iv` buffer - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_cipher_set_iv_t)(void *op_context, - const uint8_t *p_iv, - size_t iv_length); - -/** \brief A function that continues a previously started secure element cipher - * operation - * - * \param[in,out] op_context A hardware-specific structure for the - * previously started cipher operation - * \param[in] p_input A buffer containing the data to be - * encrypted/decrypted - * \param[in] input_size The size in bytes of the buffer pointed to - * by `p_input` - * \param[out] p_output The caller-allocated buffer where the - * output will be placed - * \param[in] output_size The allocated size in bytes of the - * `p_output` buffer - * \param[out] p_output_length After completion, will contain the number - * of bytes placed in the `p_output` buffer - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_cipher_update_t)(void *op_context, - const uint8_t *p_input, - size_t input_size, - uint8_t *p_output, - size_t output_size, - size_t *p_output_length); - -/** \brief A function that completes a previously started secure element cipher - * operation - * - * \param[in,out] op_context A hardware-specific structure for the - * previously started cipher operation - * \param[out] p_output The caller-allocated buffer where the output - * will be placed - * \param[in] output_size The allocated size in bytes of the `p_output` - * buffer - * \param[out] p_output_length After completion, will contain the number of - * bytes placed in the `p_output` buffer - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_cipher_finish_t)(void *op_context, - uint8_t *p_output, - size_t output_size, - size_t *p_output_length); - -/** \brief A function that aborts a previously started secure element cipher - * operation - * - * \param[in,out] op_context A hardware-specific structure for the - * previously started cipher operation - */ -typedef psa_status_t (*psa_drv_se_cipher_abort_t)(void *op_context); - -/** \brief A function that performs the ECB block mode for secure element - * cipher operations - * - * Note: this function should only be used with implementations that do not - * provide a needed higher-level operation. - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot The slot of the key to be used for the operation - * \param[in] algorithm The algorithm to be used in the cipher operation - * \param[in] direction Indicates whether the operation is an encrypt or - * decrypt - * \param[in] p_input A buffer containing the data to be - * encrypted/decrypted - * \param[in] input_size The size in bytes of the buffer pointed to by - * `p_input` - * \param[out] p_output The caller-allocated buffer where the output - * will be placed - * \param[in] output_size The allocated size in bytes of the `p_output` - * buffer - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - */ -typedef psa_status_t (*psa_drv_se_cipher_ecb_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t algorithm, - psa_encrypt_or_decrypt_t direction, - const uint8_t *p_input, - size_t input_size, - uint8_t *p_output, - size_t output_size); - -/** - * \brief A struct containing all of the function pointers needed to implement - * cipher operations using secure elements. - * - * PSA Crypto API implementations should populate instances of the table as - * appropriate upon startup or at build time. - * - * If one of the functions is not implemented (such as - * `psa_drv_se_cipher_ecb_t`), it should be set to NULL. - */ -typedef struct { - /** The size in bytes of the hardware-specific secure element cipher - * context structure - */ - size_t MBEDTLS_PRIVATE(context_size); - /** Function that performs a cipher setup operation */ - psa_drv_se_cipher_setup_t MBEDTLS_PRIVATE(p_setup); - /** Function that sets a cipher IV (if necessary) */ - psa_drv_se_cipher_set_iv_t MBEDTLS_PRIVATE(p_set_iv); - /** Function that performs a cipher update operation */ - psa_drv_se_cipher_update_t MBEDTLS_PRIVATE(p_update); - /** Function that completes a cipher operation */ - psa_drv_se_cipher_finish_t MBEDTLS_PRIVATE(p_finish); - /** Function that aborts a cipher operation */ - psa_drv_se_cipher_abort_t MBEDTLS_PRIVATE(p_abort); - /** Function that performs ECB mode for a cipher operation - * (Danger: ECB mode should not be used directly by clients of the PSA - * Crypto Client API) - */ - psa_drv_se_cipher_ecb_t MBEDTLS_PRIVATE(p_ecb); -} psa_drv_se_cipher_t; - -/**@}*/ - -/** \defgroup se_asymmetric Secure Element Asymmetric Cryptography - * - * Since the amount of data that can (or should) be encrypted or signed using - * asymmetric keys is limited by the key size, asymmetric key operations using - * keys in a secure element must be done in single function calls. - */ -/**@{*/ - -/** - * \brief A function that signs a hash or short message with a private key in - * a secure element - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Key slot of an asymmetric key pair - * \param[in] alg A signature algorithm that is compatible - * with the type of `key` - * \param[in] p_hash The hash to sign - * \param[in] hash_length Size of the `p_hash` buffer in bytes - * \param[out] p_signature Buffer where the signature is to be written - * \param[in] signature_size Size of the `p_signature` buffer in bytes - * \param[out] p_signature_length On success, the number of bytes - * that make up the returned signature value - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_asymmetric_sign_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - const uint8_t *p_hash, - size_t hash_length, - uint8_t *p_signature, - size_t signature_size, - size_t *p_signature_length); - -/** - * \brief A function that verifies the signature a hash or short message using - * an asymmetric public key in a secure element - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Key slot of a public key or an asymmetric key - * pair - * \param[in] alg A signature algorithm that is compatible with - * the type of `key` - * \param[in] p_hash The hash whose signature is to be verified - * \param[in] hash_length Size of the `p_hash` buffer in bytes - * \param[in] p_signature Buffer containing the signature to verify - * \param[in] signature_length Size of the `p_signature` buffer in bytes - * - * \retval #PSA_SUCCESS - * The signature is valid. - */ -typedef psa_status_t (*psa_drv_se_asymmetric_verify_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - const uint8_t *p_hash, - size_t hash_length, - const uint8_t *p_signature, - size_t signature_length); - -/** - * \brief A function that encrypts a short message with an asymmetric public - * key in a secure element - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Key slot of a public key or an asymmetric key - * pair - * \param[in] alg An asymmetric encryption algorithm that is - * compatible with the type of `key` - * \param[in] p_input The message to encrypt - * \param[in] input_length Size of the `p_input` buffer in bytes - * \param[in] p_salt A salt or label, if supported by the - * encryption algorithm - * If the algorithm does not support a - * salt, pass `NULL`. - * If the algorithm supports an optional - * salt and you do not want to pass a salt, - * pass `NULL`. - * For #PSA_ALG_RSA_PKCS1V15_CRYPT, no salt is - * supported. - * \param[in] salt_length Size of the `p_salt` buffer in bytes - * If `p_salt` is `NULL`, pass 0. - * \param[out] p_output Buffer where the encrypted message is to - * be written - * \param[in] output_size Size of the `p_output` buffer in bytes - * \param[out] p_output_length On success, the number of bytes that make up - * the returned output - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_asymmetric_encrypt_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - const uint8_t *p_input, - size_t input_length, - const uint8_t *p_salt, - size_t salt_length, - uint8_t *p_output, - size_t output_size, - size_t *p_output_length); - -/** - * \brief A function that decrypts a short message with an asymmetric private - * key in a secure element. - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Key slot of an asymmetric key pair - * \param[in] alg An asymmetric encryption algorithm that is - * compatible with the type of `key` - * \param[in] p_input The message to decrypt - * \param[in] input_length Size of the `p_input` buffer in bytes - * \param[in] p_salt A salt or label, if supported by the - * encryption algorithm - * If the algorithm does not support a - * salt, pass `NULL`. - * If the algorithm supports an optional - * salt and you do not want to pass a salt, - * pass `NULL`. - * For #PSA_ALG_RSA_PKCS1V15_CRYPT, no salt is - * supported. - * \param[in] salt_length Size of the `p_salt` buffer in bytes - * If `p_salt` is `NULL`, pass 0. - * \param[out] p_output Buffer where the decrypted message is to - * be written - * \param[in] output_size Size of the `p_output` buffer in bytes - * \param[out] p_output_length On success, the number of bytes - * that make up the returned output - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_asymmetric_decrypt_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t alg, - const uint8_t *p_input, - size_t input_length, - const uint8_t *p_salt, - size_t salt_length, - uint8_t *p_output, - size_t output_size, - size_t *p_output_length); - -/** - * \brief A struct containing all of the function pointers needed to implement - * asymmetric cryptographic operations using secure elements. - * - * PSA Crypto API implementations should populate instances of the table as - * appropriate upon startup or at build time. - * - * If one of the functions is not implemented, it should be set to NULL. - */ -typedef struct { - /** Function that performs an asymmetric sign operation */ - psa_drv_se_asymmetric_sign_t MBEDTLS_PRIVATE(p_sign); - /** Function that performs an asymmetric verify operation */ - psa_drv_se_asymmetric_verify_t MBEDTLS_PRIVATE(p_verify); - /** Function that performs an asymmetric encrypt operation */ - psa_drv_se_asymmetric_encrypt_t MBEDTLS_PRIVATE(p_encrypt); - /** Function that performs an asymmetric decrypt operation */ - psa_drv_se_asymmetric_decrypt_t MBEDTLS_PRIVATE(p_decrypt); -} psa_drv_se_asymmetric_t; - -/**@}*/ - -/** \defgroup se_aead Secure Element Authenticated Encryption with Additional Data - * Authenticated Encryption with Additional Data (AEAD) operations with secure - * elements must be done in one function call. While this creates a burden for - * implementers as there must be sufficient space in memory for the entire - * message, it prevents decrypted data from being made available before the - * authentication operation is complete and the data is known to be authentic. - */ -/**@{*/ - -/** \brief A function that performs a secure element authenticated encryption - * operation - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Slot containing the key to use. - * \param[in] algorithm The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(`alg`) is true) - * \param[in] p_nonce Nonce or IV to use - * \param[in] nonce_length Size of the `p_nonce` buffer in bytes - * \param[in] p_additional_data Additional data that will be - * authenticated but not encrypted - * \param[in] additional_data_length Size of `p_additional_data` in bytes - * \param[in] p_plaintext Data that will be authenticated and - * encrypted - * \param[in] plaintext_length Size of `p_plaintext` in bytes - * \param[out] p_ciphertext Output buffer for the authenticated and - * encrypted data. The additional data is - * not part of this output. For algorithms - * where the encrypted data and the - * authentication tag are defined as - * separate outputs, the authentication - * tag is appended to the encrypted data. - * \param[in] ciphertext_size Size of the `p_ciphertext` buffer in - * bytes - * \param[out] p_ciphertext_length On success, the size of the output in - * the `p_ciphertext` buffer - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_aead_encrypt_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t algorithm, - const uint8_t *p_nonce, - size_t nonce_length, - const uint8_t *p_additional_data, - size_t additional_data_length, - const uint8_t *p_plaintext, - size_t plaintext_length, - uint8_t *p_ciphertext, - size_t ciphertext_size, - size_t *p_ciphertext_length); - -/** A function that performs a secure element authenticated decryption operation - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key_slot Slot containing the key to use - * \param[in] algorithm The AEAD algorithm to compute - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(`alg`) is true) - * \param[in] p_nonce Nonce or IV to use - * \param[in] nonce_length Size of the `p_nonce` buffer in bytes - * \param[in] p_additional_data Additional data that has been - * authenticated but not encrypted - * \param[in] additional_data_length Size of `p_additional_data` in bytes - * \param[in] p_ciphertext Data that has been authenticated and - * encrypted. - * For algorithms where the encrypted data - * and the authentication tag are defined - * as separate inputs, the buffer must - * contain the encrypted data followed by - * the authentication tag. - * \param[in] ciphertext_length Size of `p_ciphertext` in bytes - * \param[out] p_plaintext Output buffer for the decrypted data - * \param[in] plaintext_size Size of the `p_plaintext` buffer in - * bytes - * \param[out] p_plaintext_length On success, the size of the output in - * the `p_plaintext` buffer - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_aead_decrypt_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - psa_algorithm_t algorithm, - const uint8_t *p_nonce, - size_t nonce_length, - const uint8_t *p_additional_data, - size_t additional_data_length, - const uint8_t *p_ciphertext, - size_t ciphertext_length, - uint8_t *p_plaintext, - size_t plaintext_size, - size_t *p_plaintext_length); - -/** - * \brief A struct containing all of the function pointers needed to implement - * secure element Authenticated Encryption with Additional Data operations - * - * PSA Crypto API implementations should populate instances of the table as - * appropriate upon startup. - * - * If one of the functions is not implemented, it should be set to NULL. - */ -typedef struct { - /** Function that performs the AEAD encrypt operation */ - psa_drv_se_aead_encrypt_t MBEDTLS_PRIVATE(p_encrypt); - /** Function that performs the AEAD decrypt operation */ - psa_drv_se_aead_decrypt_t MBEDTLS_PRIVATE(p_decrypt); -} psa_drv_se_aead_t; -/**@}*/ - -/** \defgroup se_key_management Secure Element Key Management - * Currently, key management is limited to importing keys in the clear, - * destroying keys, and exporting keys in the clear. - * Whether a key may be exported is determined by the key policies in place - * on the key slot. - */ -/**@{*/ - -/** An enumeration indicating how a key is created. - */ -typedef enum { - PSA_KEY_CREATION_IMPORT, /**< During psa_import_key() */ - PSA_KEY_CREATION_GENERATE, /**< During psa_generate_key() */ - PSA_KEY_CREATION_DERIVE, /**< During psa_key_derivation_output_key() */ - PSA_KEY_CREATION_COPY, /**< During psa_copy_key() */ - -#ifndef __DOXYGEN_ONLY__ - /** A key is being registered with mbedtls_psa_register_se_key(). - * - * The core only passes this value to - * psa_drv_se_key_management_t::p_validate_slot_number, not to - * psa_drv_se_key_management_t::p_allocate. The call to - * `p_validate_slot_number` is not followed by any other call to the - * driver: the key is considered successfully registered if the call to - * `p_validate_slot_number` succeeds, or if `p_validate_slot_number` is - * null. - * - * With this creation method, the driver must return #PSA_SUCCESS if - * the given attributes are compatible with the existing key in the slot, - * and #PSA_ERROR_DOES_NOT_EXIST if the driver can determine that there - * is no key with the specified slot number. - * - * This is an Mbed TLS extension. - */ - PSA_KEY_CREATION_REGISTER, -#endif -} psa_key_creation_method_t; - -/** \brief A function that allocates a slot for a key. - * - * To create a key in a specific slot in a secure element, the core - * first calls this function to determine a valid slot number, - * then calls a function to create the key material in that slot. - * In nominal conditions (that is, if no error occurs), - * the effect of a call to a key creation function in the PSA Cryptography - * API with a lifetime that places the key in a secure element is the - * following: - * -# The core calls psa_drv_se_key_management_t::p_allocate - * (or in some implementations - * psa_drv_se_key_management_t::p_validate_slot_number). The driver - * selects (or validates) a suitable slot number given the key attributes - * and the state of the secure element. - * -# The core calls a key creation function in the driver. - * - * The key creation functions in the PSA Cryptography API are: - * - psa_import_key(), which causes - * a call to `p_allocate` with \p method = #PSA_KEY_CREATION_IMPORT - * then a call to psa_drv_se_key_management_t::p_import. - * - psa_generate_key(), which causes - * a call to `p_allocate` with \p method = #PSA_KEY_CREATION_GENERATE - * then a call to psa_drv_se_key_management_t::p_import. - * - psa_key_derivation_output_key(), which causes - * a call to `p_allocate` with \p method = #PSA_KEY_CREATION_DERIVE - * then a call to psa_drv_se_key_derivation_t::p_derive. - * - psa_copy_key(), which causes - * a call to `p_allocate` with \p method = #PSA_KEY_CREATION_COPY - * then a call to psa_drv_se_key_management_t::p_export. - * - * In case of errors, other behaviors are possible. - * - If the PSA Cryptography subsystem dies after the first step, - * for example because the device has lost power abruptly, - * the second step may never happen, or may happen after a reset - * and re-initialization. Alternatively, after a reset and - * re-initialization, the core may call - * psa_drv_se_key_management_t::p_destroy on the slot number that - * was allocated (or validated) instead of calling a key creation function. - * - If an error occurs, the core may call - * psa_drv_se_key_management_t::p_destroy on the slot number that - * was allocated (or validated) instead of calling a key creation function. - * - * Errors and system resets also have an impact on the driver's persistent - * data. If a reset happens before the overall key creation process is - * completed (before or after the second step above), it is unspecified - * whether the persistent data after the reset is identical to what it - * was before or after the call to `p_allocate` (or `p_validate_slot_number`). - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] persistent_data A pointer to the persistent data - * that allows writing. - * \param[in] attributes Attributes of the key. - * \param method The way in which the key is being created. - * \param[out] key_slot Slot where the key will be stored. - * This must be a valid slot for a key of the - * chosen type. It must be unoccupied. - * - * \retval #PSA_SUCCESS - * Success. - * The core will record \c *key_slot as the key slot where the key - * is stored and will update the persistent data in storage. - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_INSUFFICIENT_STORAGE \emptydescription - */ -typedef psa_status_t (*psa_drv_se_allocate_key_t)( - psa_drv_se_context_t *drv_context, - void *persistent_data, - const psa_key_attributes_t *attributes, - psa_key_creation_method_t method, - psa_key_slot_number_t *key_slot); - -/** \brief A function that determines whether a slot number is valid - * for a key. - * - * To create a key in a specific slot in a secure element, the core - * first calls this function to validate the choice of slot number, - * then calls a function to create the key material in that slot. - * See the documentation of #psa_drv_se_allocate_key_t for more details. - * - * As of the PSA Cryptography API specification version 1.0, there is no way - * for applications to trigger a call to this function. However some - * implementations offer the capability to create or declare a key in - * a specific slot via implementation-specific means, generally for the - * sake of initial device provisioning or onboarding. Such a mechanism may - * be added to a future version of the PSA Cryptography API specification. - * - * This function may update the driver's persistent data through - * \p persistent_data. The core will save the updated persistent data at the - * end of the key creation process. See the description of - * ::psa_drv_se_allocate_key_t for more information. - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] persistent_data A pointer to the persistent data - * that allows writing. - * \param[in] attributes Attributes of the key. - * \param method The way in which the key is being created. - * \param[in] key_slot Slot where the key is to be stored. - * - * \retval #PSA_SUCCESS - * The given slot number is valid for a key with the given - * attributes. - * \retval #PSA_ERROR_INVALID_ARGUMENT - * The given slot number is not valid for a key with the - * given attributes. This includes the case where the slot - * number is not valid at all. - * \retval #PSA_ERROR_ALREADY_EXISTS - * There is already a key with the specified slot number. - * Drivers may choose to return this error from the key - * creation function instead. - */ -typedef psa_status_t (*psa_drv_se_validate_slot_number_t)( - psa_drv_se_context_t *drv_context, - void *persistent_data, - const psa_key_attributes_t *attributes, - psa_key_creation_method_t method, - psa_key_slot_number_t key_slot); - -/** \brief A function that imports a key into a secure element in binary format - * - * This function can support any output from psa_export_key(). Refer to the - * documentation of psa_export_key() for the format for each key type. - * - * \param[in,out] drv_context The driver context structure. - * \param key_slot Slot where the key will be stored. - * This must be a valid slot for a key of the - * chosen type. It must be unoccupied. - * \param[in] attributes The key attributes, including the lifetime, - * the key type and the usage policy. - * Drivers should not access the key size stored - * in the attributes: it may not match the - * data passed in \p data. - * Drivers can call psa_get_key_lifetime(), - * psa_get_key_type(), - * psa_get_key_usage_flags() and - * psa_get_key_algorithm() to access this - * information. - * \param[in] data Buffer containing the key data. - * \param[in] data_length Size of the \p data buffer in bytes. - * \param[out] bits On success, the key size in bits. The driver - * must determine this value after parsing the - * key according to the key type. - * This value is not used if the function fails. - * - * \retval #PSA_SUCCESS - * Success. - */ -typedef psa_status_t (*psa_drv_se_import_key_t)( - psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - const psa_key_attributes_t *attributes, - const uint8_t *data, - size_t data_length, - size_t *bits); - -/** - * \brief A function that destroys a secure element key and restore the slot to - * its default state - * - * This function destroys the content of the key from a secure element. - * Implementations shall make a best effort to ensure that any previous content - * of the slot is unrecoverable. - * - * This function returns the specified slot to its default state. - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] persistent_data A pointer to the persistent data - * that allows writing. - * \param key_slot The key slot to erase. - * - * \retval #PSA_SUCCESS - * The slot's content, if any, has been erased. - */ -typedef psa_status_t (*psa_drv_se_destroy_key_t)( - psa_drv_se_context_t *drv_context, - void *persistent_data, - psa_key_slot_number_t key_slot); - -/** - * \brief A function that exports a secure element key in binary format - * - * The output of this function can be passed to psa_import_key() to - * create an equivalent object. - * - * If a key is created with `psa_import_key()` and then exported with - * this function, it is not guaranteed that the resulting data is - * identical: the implementation may choose a different representation - * of the same key if the format permits it. - * - * This function should generate output in the same format that - * `psa_export_key()` does. Refer to the - * documentation of `psa_export_key()` for the format for each key type. - * - * \param[in,out] drv_context The driver context structure. - * \param[in] key Slot whose content is to be exported. This must - * be an occupied key slot. - * \param[out] p_data Buffer where the key data is to be written. - * \param[in] data_size Size of the `p_data` buffer in bytes. - * \param[out] p_data_length On success, the number of bytes - * that make up the key data. - * - * \retval #PSA_SUCCESS \emptydescription - * \retval #PSA_ERROR_DOES_NOT_EXIST \emptydescription - * \retval #PSA_ERROR_NOT_PERMITTED \emptydescription - * \retval #PSA_ERROR_NOT_SUPPORTED \emptydescription - * \retval #PSA_ERROR_COMMUNICATION_FAILURE \emptydescription - * \retval #PSA_ERROR_HARDWARE_FAILURE \emptydescription - * \retval #PSA_ERROR_CORRUPTION_DETECTED \emptydescription - */ -typedef psa_status_t (*psa_drv_se_export_key_t)(psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key, - uint8_t *p_data, - size_t data_size, - size_t *p_data_length); - -/** - * \brief A function that generates a symmetric or asymmetric key on a secure - * element - * - * If the key type \c type recorded in \p attributes - * is asymmetric (#PSA_KEY_TYPE_IS_ASYMMETRIC(\c type) = 1), - * the driver may export the public key at the time of generation, - * in the format documented for psa_export_public_key() by writing it - * to the \p pubkey buffer. - * This is optional, intended for secure elements that output the - * public key at generation time and that cannot export the public key - * later. Drivers that do not need this feature should leave - * \p *pubkey_length set to 0 and should - * implement the psa_drv_key_management_t::p_export_public function. - * Some implementations do not support this feature, in which case - * \p pubkey is \c NULL and \p pubkey_size is 0. - * - * \param[in,out] drv_context The driver context structure. - * \param key_slot Slot where the key will be stored. - * This must be a valid slot for a key of the - * chosen type. It must be unoccupied. - * \param[in] attributes The key attributes, including the lifetime, - * the key type and size, and the usage policy. - * Drivers can call psa_get_key_lifetime(), - * psa_get_key_type(), psa_get_key_bits(), - * psa_get_key_usage_flags() and - * psa_get_key_algorithm() to access this - * information. - * \param[out] pubkey A buffer where the driver can write the - * public key, when generating an asymmetric - * key pair. - * This is \c NULL when generating a symmetric - * key or if the core does not support - * exporting the public key at generation time. - * \param pubkey_size The size of the `pubkey` buffer in bytes. - * This is 0 when generating a symmetric - * key or if the core does not support - * exporting the public key at generation time. - * \param[out] pubkey_length On entry, this is always 0. - * On success, the number of bytes written to - * \p pubkey. If this is 0 or unchanged on return, - * the core will not read the \p pubkey buffer, - * and will instead call the driver's - * psa_drv_key_management_t::p_export_public - * function to export the public key when needed. - */ -typedef psa_status_t (*psa_drv_se_generate_key_t)( - psa_drv_se_context_t *drv_context, - psa_key_slot_number_t key_slot, - const psa_key_attributes_t *attributes, - uint8_t *pubkey, size_t pubkey_size, size_t *pubkey_length); - -/** - * \brief A struct containing all of the function pointers needed to for secure - * element key management - * - * PSA Crypto API implementations should populate instances of the table as - * appropriate upon startup or at build time. - * - * If one of the functions is not implemented, it should be set to NULL. - */ -typedef struct { - /** Function that allocates a slot for a key. */ - psa_drv_se_allocate_key_t MBEDTLS_PRIVATE(p_allocate); - /** Function that checks the validity of a slot for a key. */ - psa_drv_se_validate_slot_number_t MBEDTLS_PRIVATE(p_validate_slot_number); - /** Function that performs a key import operation */ - psa_drv_se_import_key_t MBEDTLS_PRIVATE(p_import); - /** Function that performs a generation */ - psa_drv_se_generate_key_t MBEDTLS_PRIVATE(p_generate); - /** Function that performs a key destroy operation */ - psa_drv_se_destroy_key_t MBEDTLS_PRIVATE(p_destroy); - /** Function that performs a key export operation */ - psa_drv_se_export_key_t MBEDTLS_PRIVATE(p_export); - /** Function that performs a public key export operation */ - psa_drv_se_export_key_t MBEDTLS_PRIVATE(p_export_public); -} psa_drv_se_key_management_t; - -/**@}*/ - -/** \defgroup driver_derivation Secure Element Key Derivation and Agreement - * Key derivation is the process of generating new key material using an - * existing key and additional parameters, iterating through a basic - * cryptographic function, such as a hash. - * Key agreement is a part of cryptographic protocols that allows two parties - * to agree on the same key value, but starting from different original key - * material. - * The flows are similar, and the PSA Crypto Driver Model uses the same functions - * for both of the flows. - * - * There are two different final functions for the flows, - * `psa_drv_se_key_derivation_derive` and `psa_drv_se_key_derivation_export`. - * `psa_drv_se_key_derivation_derive` is used when the key material should be - * placed in a slot on the hardware and not exposed to the caller. - * `psa_drv_se_key_derivation_export` is used when the key material should be - * returned to the PSA Cryptographic API implementation. - * - * Different key derivation algorithms require a different number of inputs. - * Instead of having an API that takes as input variable length arrays, which - * can be problematic to manage on embedded platforms, the inputs are passed - * to the driver via a function, `psa_drv_se_key_derivation_collateral`, that - * is called multiple times with different `collateral_id`s. Thus, for a key - * derivation algorithm that required 3 parameter inputs, the flow would look - * something like: - * ~~~~~~~~~~~~~{.c} - * psa_drv_se_key_derivation_setup(kdf_algorithm, source_key, dest_key_size_bytes); - * psa_drv_se_key_derivation_collateral(kdf_algorithm_collateral_id_0, - * p_collateral_0, - * collateral_0_size); - * psa_drv_se_key_derivation_collateral(kdf_algorithm_collateral_id_1, - * p_collateral_1, - * collateral_1_size); - * psa_drv_se_key_derivation_collateral(kdf_algorithm_collateral_id_2, - * p_collateral_2, - * collateral_2_size); - * psa_drv_se_key_derivation_derive(); - * ~~~~~~~~~~~~~ - * - * key agreement example: - * ~~~~~~~~~~~~~{.c} - * psa_drv_se_key_derivation_setup(alg, source_key. dest_key_size_bytes); - * psa_drv_se_key_derivation_collateral(DHE_PUBKEY, p_pubkey, pubkey_size); - * psa_drv_se_key_derivation_export(p_session_key, - * session_key_size, - * &session_key_length); - * ~~~~~~~~~~~~~ - */ -/**@{*/ - -/** \brief A function that Sets up a secure element key derivation operation by - * specifying the algorithm and the source key sot - * - * \param[in,out] drv_context The driver context structure. - * \param[in,out] op_context A hardware-specific structure containing any - * context information for the implementation - * \param[in] kdf_alg The algorithm to be used for the key derivation - * \param[in] source_key The key to be used as the source material for - * the key derivation - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_key_derivation_setup_t)(psa_drv_se_context_t *drv_context, - void *op_context, - psa_algorithm_t kdf_alg, - psa_key_slot_number_t source_key); - -/** \brief A function that provides collateral (parameters) needed for a secure - * element key derivation or key agreement operation - * - * Since many key derivation algorithms require multiple parameters, it is - * expected that this function may be called multiple times for the same - * operation, each with a different algorithm-specific `collateral_id` - * - * \param[in,out] op_context A hardware-specific structure containing any - * context information for the implementation - * \param[in] collateral_id An ID for the collateral being provided - * \param[in] p_collateral A buffer containing the collateral data - * \param[in] collateral_size The size in bytes of the collateral - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_key_derivation_collateral_t)(void *op_context, - uint32_t collateral_id, - const uint8_t *p_collateral, - size_t collateral_size); - -/** \brief A function that performs the final secure element key derivation - * step and place the generated key material in a slot - * - * \param[in,out] op_context A hardware-specific structure containing any - * context information for the implementation - * \param[in] dest_key The slot where the generated key material - * should be placed - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_key_derivation_derive_t)(void *op_context, - psa_key_slot_number_t dest_key); - -/** \brief A function that performs the final step of a secure element key - * agreement and place the generated key material in a buffer - * - * \param[out] p_output Buffer in which to place the generated key - * material - * \param[in] output_size The size in bytes of `p_output` - * \param[out] p_output_length Upon success, contains the number of bytes of - * key material placed in `p_output` - * - * \retval #PSA_SUCCESS \emptydescription - */ -typedef psa_status_t (*psa_drv_se_key_derivation_export_t)(void *op_context, - uint8_t *p_output, - size_t output_size, - size_t *p_output_length); - -/** - * \brief A struct containing all of the function pointers needed to for secure - * element key derivation and agreement - * - * PSA Crypto API implementations should populate instances of the table as - * appropriate upon startup. - * - * If one of the functions is not implemented, it should be set to NULL. - */ -typedef struct { - /** The driver-specific size of the key derivation context */ - size_t MBEDTLS_PRIVATE(context_size); - /** Function that performs a key derivation setup */ - psa_drv_se_key_derivation_setup_t MBEDTLS_PRIVATE(p_setup); - /** Function that sets key derivation collateral */ - psa_drv_se_key_derivation_collateral_t MBEDTLS_PRIVATE(p_collateral); - /** Function that performs a final key derivation step */ - psa_drv_se_key_derivation_derive_t MBEDTLS_PRIVATE(p_derive); - /** Function that performs a final key derivation or agreement and - * exports the key */ - psa_drv_se_key_derivation_export_t MBEDTLS_PRIVATE(p_export); -} psa_drv_se_key_derivation_t; - -/**@}*/ - -/** \defgroup se_registration Secure element driver registration - */ -/**@{*/ - -/** A structure containing pointers to all the entry points of a - * secure element driver. - * - * Future versions of this specification may add extra substructures at - * the end of this structure. - */ -typedef struct { - /** The version of the driver HAL that this driver implements. - * This is a protection against loading driver binaries built against - * a different version of this specification. - * Use #PSA_DRV_SE_HAL_VERSION. - */ - uint32_t MBEDTLS_PRIVATE(hal_version); - - /** The size of the driver's persistent data in bytes. - * - * This can be 0 if the driver does not need persistent data. - * - * See the documentation of psa_drv_se_context_t::persistent_data - * for more information about why and how a driver can use - * persistent data. - */ - size_t MBEDTLS_PRIVATE(persistent_data_size); - - /** The driver initialization function. - * - * This function is called once during the initialization of the - * PSA Cryptography subsystem, before any other function of the - * driver is called. If this function returns a failure status, - * the driver will be unusable, at least until the next system reset. - * - * If this field is \c NULL, it is equivalent to a function that does - * nothing and returns #PSA_SUCCESS. - */ - psa_drv_se_init_t MBEDTLS_PRIVATE(p_init); - - const psa_drv_se_key_management_t *MBEDTLS_PRIVATE(key_management); - const psa_drv_se_mac_t *MBEDTLS_PRIVATE(mac); - const psa_drv_se_cipher_t *MBEDTLS_PRIVATE(cipher); - const psa_drv_se_aead_t *MBEDTLS_PRIVATE(aead); - const psa_drv_se_asymmetric_t *MBEDTLS_PRIVATE(asymmetric); - const psa_drv_se_key_derivation_t *MBEDTLS_PRIVATE(derivation); -} psa_drv_se_t; - -/** The current version of the secure element driver HAL. - */ -/* 0.0.0 patchlevel 5 */ -#define PSA_DRV_SE_HAL_VERSION 0x00000005 - -/** Register an external cryptoprocessor (secure element) driver. - * - * This function is only intended to be used by driver code, not by - * application code. In implementations with separation between the - * PSA cryptography module and applications, this function should - * only be available to callers that run in the same memory space as - * the cryptography module, and should not be exposed to applications - * running in a different memory space. - * - * This function may be called before psa_crypto_init(). It is - * implementation-defined whether this function may be called - * after psa_crypto_init(). - * - * \note Implementations store metadata about keys including the lifetime - * value, which contains the driver's location indicator. Therefore, - * from one instantiation of the PSA Cryptography - * library to the next one, if there is a key in storage with a certain - * lifetime value, you must always register the same driver (or an - * updated version that communicates with the same secure element) - * with the same location value. - * - * \param location The location value through which this driver will - * be exposed to applications. - * This driver will be used for all keys such that - * `location == #PSA_KEY_LIFETIME_GET_LOCATION( lifetime )`. - * The value #PSA_KEY_LOCATION_LOCAL_STORAGE is reserved - * and may not be used for drivers. Implementations - * may reserve other values. - * \param[in] methods The method table of the driver. This structure must - * remain valid for as long as the cryptography - * module keeps running. It is typically a global - * constant. - * - * \return #PSA_SUCCESS - * The driver was successfully registered. Applications can now - * use \p location to access keys through the methods passed to - * this function. - * \return #PSA_ERROR_BAD_STATE - * This function was called after the initialization of the - * cryptography module, and this implementation does not support - * driver registration at this stage. - * \return #PSA_ERROR_ALREADY_EXISTS - * There is already a registered driver for this value of \p location. - * \return #PSA_ERROR_INVALID_ARGUMENT - * \p location is a reserved value. - * \return #PSA_ERROR_NOT_SUPPORTED - * `methods->hal_version` is not supported by this implementation. - * \return #PSA_ERROR_INSUFFICIENT_MEMORY - * \return #PSA_ERROR_NOT_PERMITTED - * \return #PSA_ERROR_STORAGE_FAILURE - * \return #PSA_ERROR_DATA_CORRUPT - */ -psa_status_t psa_register_se_driver( - psa_key_location_t location, - const psa_drv_se_t *methods); - -/**@}*/ - -#ifdef __cplusplus -} -#endif - -#endif /* PSA_CRYPTO_SE_DRIVER_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_sizes.h b/android/mbedtls/arm64-v8a/include/psa/crypto_sizes.h deleted file mode 100644 index 87b8c39fa..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_sizes.h +++ /dev/null @@ -1,1317 +0,0 @@ -/** - * \file psa/crypto_sizes.h - * - * \brief PSA cryptography module: Mbed TLS buffer size macros - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * This file contains the definitions of macros that are useful to - * compute buffer sizes. The signatures and semantics of these macros - * are standardized, but the definitions are not, because they depend on - * the available algorithms and, in some cases, on permitted tolerances - * on buffer sizes. - * - * In implementations with isolation between the application and the - * cryptography module, implementers should take care to ensure that - * the definitions that are exposed to applications match what the - * module implements. - * - * Macros that compute sizes whose values do not depend on the - * implementation are in crypto.h. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_SIZES_H -#define PSA_CRYPTO_SIZES_H - -/* - * Include the build-time configuration information header. Here, we do not - * include `"mbedtls/build_info.h"` directly but `"psa/build_info.h"`, which - * is basically just an alias to it. This is to ease the maintenance of the - * TF-PSA-Crypto repository which has a different build system and - * configuration. - */ -#include "psa/build_info.h" - -#define PSA_BITS_TO_BYTES(bits) (((bits) + 7u) / 8u) -#define PSA_BYTES_TO_BITS(bytes) ((bytes) * 8u) -#define PSA_MAX_OF_THREE(a, b, c) ((a) <= (b) ? (b) <= (c) ? \ - (c) : (b) : (a) <= (c) ? (c) : (a)) - -#define PSA_ROUND_UP_TO_MULTIPLE(block_size, length) \ - (((length) + (block_size) - 1) / (block_size) * (block_size)) - -/** The size of the output of psa_hash_finish(), in bytes. - * - * This is also the hash size that psa_hash_verify() expects. - * - * \param alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p alg) is true), or an HMAC algorithm - * (#PSA_ALG_HMAC(\c hash_alg) where \c hash_alg is a - * hash algorithm). - * - * \return The hash size for the specified hash algorithm. - * If the hash algorithm is not recognized, return 0. - */ -#define PSA_HASH_LENGTH(alg) \ - ( \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_MD5 ? 16u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_RIPEMD160 ? 20u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_1 ? 20u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_224 ? 28u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_256 ? 32u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_384 ? 48u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512_224 ? 28u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512_256 ? 32u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_224 ? 28u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_256 ? 32u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_384 ? 48u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_512 ? 64u : \ - 0u) - -/** The input block size of a hash algorithm, in bytes. - * - * Hash algorithms process their input data in blocks. Hash operations will - * retain any partial blocks until they have enough input to fill the block or - * until the operation is finished. - * This affects the output from psa_hash_suspend(). - * - * \param alg A hash algorithm (\c PSA_ALG_XXX value such that - * PSA_ALG_IS_HASH(\p alg) is true). - * - * \return The block size in bytes for the specified hash algorithm. - * If the hash algorithm is not recognized, return 0. - * An implementation can return either 0 or the correct size for a - * hash algorithm that it recognizes, but does not support. - */ -#define PSA_HASH_BLOCK_LENGTH(alg) \ - ( \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_MD5 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_RIPEMD160 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_1 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_224 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_256 ? 64u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_384 ? 128u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512 ? 128u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512_224 ? 128u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA_512_256 ? 128u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_224 ? 144u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_256 ? 136u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_384 ? 104u : \ - PSA_ALG_HMAC_GET_HASH(alg) == PSA_ALG_SHA3_512 ? 72u : \ - 0u) - -/** \def PSA_HASH_MAX_SIZE - * - * Maximum size of a hash. - * - * This macro expands to a compile-time constant integer. This value - * is the maximum size of a hash in bytes. - */ -/* Note: for HMAC-SHA-3, the block size is 144 bytes for HMAC-SHA3-224, - * 136 bytes for HMAC-SHA3-256, 104 bytes for SHA3-384, 72 bytes for - * HMAC-SHA3-512. */ -/* Note: PSA_HASH_MAX_SIZE should be kept in sync with MBEDTLS_MD_MAX_SIZE, - * see the note on MBEDTLS_MD_MAX_SIZE for details. */ -#if defined(PSA_WANT_ALG_SHA3_224) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 144u -#elif defined(PSA_WANT_ALG_SHA3_256) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 136u -#elif defined(PSA_WANT_ALG_SHA_512) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 128u -#elif defined(PSA_WANT_ALG_SHA_384) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 128u -#elif defined(PSA_WANT_ALG_SHA3_384) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 104u -#elif defined(PSA_WANT_ALG_SHA3_512) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 72u -#elif defined(PSA_WANT_ALG_SHA_256) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 64u -#elif defined(PSA_WANT_ALG_SHA_224) -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 64u -#else /* SHA-1 or smaller */ -#define PSA_HMAC_MAX_HASH_BLOCK_SIZE 64u -#endif - -#if defined(PSA_WANT_ALG_SHA_512) || defined(PSA_WANT_ALG_SHA3_512) -#define PSA_HASH_MAX_SIZE 64u -#elif defined(PSA_WANT_ALG_SHA_384) || defined(PSA_WANT_ALG_SHA3_384) -#define PSA_HASH_MAX_SIZE 48u -#elif defined(PSA_WANT_ALG_SHA_256) || defined(PSA_WANT_ALG_SHA3_256) -#define PSA_HASH_MAX_SIZE 32u -#elif defined(PSA_WANT_ALG_SHA_224) || defined(PSA_WANT_ALG_SHA3_224) -#define PSA_HASH_MAX_SIZE 28u -#else /* SHA-1 or smaller */ -#define PSA_HASH_MAX_SIZE 20u -#endif - -/** \def PSA_MAC_MAX_SIZE - * - * Maximum size of a MAC. - * - * This macro expands to a compile-time constant integer. This value - * is the maximum size of a MAC in bytes. - */ -/* All non-HMAC MACs have a maximum size that's smaller than the - * minimum possible value of PSA_HASH_MAX_SIZE in this implementation. */ -/* Note that the encoding of truncated MAC algorithms limits this value - * to 64 bytes. - */ -#define PSA_MAC_MAX_SIZE PSA_HASH_MAX_SIZE - -/** The length of a tag for an AEAD algorithm, in bytes. - * - * This macro can be used to allocate a buffer of sufficient size to store the - * tag output from psa_aead_finish(). - * - * See also #PSA_AEAD_TAG_MAX_SIZE. - * - * \param key_type The type of the AEAD key. - * \param key_bits The size of the AEAD key in bits. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \return The tag length for the specified algorithm and key. - * If the AEAD algorithm does not have an identified - * tag that can be distinguished from the rest of - * the ciphertext, return 0. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_AEAD_TAG_LENGTH(key_type, key_bits, alg) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 ? \ - PSA_ALG_AEAD_GET_TAG_LENGTH(alg) : \ - ((void) (key_bits), 0u)) - -/** The maximum tag size for all supported AEAD algorithms, in bytes. - * - * See also #PSA_AEAD_TAG_LENGTH(\p key_type, \p key_bits, \p alg). - */ -#define PSA_AEAD_TAG_MAX_SIZE 16u - -/* The maximum size of an RSA key on this implementation, in bits. - * This is a vendor-specific macro. - * - * Mbed TLS does not set a hard limit on the size of RSA keys: any key - * whose parameters fit in a bignum is accepted. However large keys can - * induce a large memory usage and long computation times. Unlike other - * auxiliary macros in this file and in crypto.h, which reflect how the - * library is configured, this macro defines how the library is - * configured. This implementation refuses to import or generate an - * RSA key whose size is larger than the value defined here. - * - * Note that an implementation may set different size limits for different - * operations, and does not need to accept all key sizes up to the limit. */ -#define PSA_VENDOR_RSA_MAX_KEY_BITS 4096u - -/* The minimum size of an RSA key on this implementation, in bits. - * This is a vendor-specific macro. - * - * Limits RSA key generation to a minimum due to avoid accidental misuse. - * This value cannot be less than 128 bits. - */ -#if defined(MBEDTLS_RSA_GEN_KEY_MIN_BITS) -#define PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS MBEDTLS_RSA_GEN_KEY_MIN_BITS -#else -#define PSA_VENDOR_RSA_GENERATE_MIN_KEY_BITS 1024 -#endif - -/* The maximum size of an DH key on this implementation, in bits. - * This is a vendor-specific macro.*/ -#if defined(PSA_WANT_DH_RFC7919_8192) -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 8192u -#elif defined(PSA_WANT_DH_RFC7919_6144) -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 6144u -#elif defined(PSA_WANT_DH_RFC7919_4096) -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 4096u -#elif defined(PSA_WANT_DH_RFC7919_3072) -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 3072u -#elif defined(PSA_WANT_DH_RFC7919_2048) -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 2048u -#else -#define PSA_VENDOR_FFDH_MAX_KEY_BITS 0u -#endif - -/* The maximum size of an ECC key on this implementation, in bits. - * This is a vendor-specific macro. */ -#if defined(PSA_WANT_ECC_SECP_R1_521) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 521u -#elif defined(PSA_WANT_ECC_BRAINPOOL_P_R1_512) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 512u -#elif defined(PSA_WANT_ECC_MONTGOMERY_448) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 448u -#elif defined(PSA_WANT_ECC_SECP_R1_384) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 384u -#elif defined(PSA_WANT_ECC_BRAINPOOL_P_R1_384) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 384u -#elif defined(PSA_WANT_ECC_SECP_R1_256) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 256u -#elif defined(PSA_WANT_ECC_SECP_K1_256) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 256u -#elif defined(PSA_WANT_ECC_BRAINPOOL_P_R1_256) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 256u -#elif defined(PSA_WANT_ECC_MONTGOMERY_255) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 255u -#elif defined(PSA_WANT_ECC_SECP_R1_224) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 224u -#elif defined(PSA_WANT_ECC_SECP_K1_224) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 224u -#elif defined(PSA_WANT_ECC_SECP_R1_192) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 192u -#elif defined(PSA_WANT_ECC_SECP_K1_192) -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 192u -#else -#define PSA_VENDOR_ECC_MAX_CURVE_BITS 0u -#endif - -/** This macro returns the maximum supported length of the PSK for the - * TLS-1.2 PSK-to-MS key derivation - * (#PSA_ALG_TLS12_PSK_TO_MS(\c hash_alg)). - * - * The maximum supported length does not depend on the chosen hash algorithm. - * - * Quoting RFC 4279, Sect 5.3: - * TLS implementations supporting these ciphersuites MUST support - * arbitrary PSK identities up to 128 octets in length, and arbitrary - * PSKs up to 64 octets in length. Supporting longer identities and - * keys is RECOMMENDED. - * - * Therefore, no implementation should define a value smaller than 64 - * for #PSA_TLS12_PSK_TO_MS_PSK_MAX_SIZE. - */ -#define PSA_TLS12_PSK_TO_MS_PSK_MAX_SIZE 128u - -/* The expected size of input passed to psa_tls12_ecjpake_to_pms_input, - * which is expected to work with P-256 curve only. */ -#define PSA_TLS12_ECJPAKE_TO_PMS_INPUT_SIZE 65u - -/* The size of a serialized K.X coordinate to be used in - * psa_tls12_ecjpake_to_pms_input. This function only accepts the P-256 - * curve. */ -#define PSA_TLS12_ECJPAKE_TO_PMS_DATA_SIZE 32u - -/* The maximum number of iterations for PBKDF2 on this implementation, in bits. - * This is a vendor-specific macro. This can be configured if necessary */ -#define PSA_VENDOR_PBKDF2_MAX_ITERATIONS 0xffffffffU - -/** The maximum size of a block cipher. */ -#define PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE 16u - -/** The size of the output of psa_mac_sign_finish(), in bytes. - * - * This is also the MAC size that psa_mac_verify_finish() expects. - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type The type of the MAC key. - * \param key_bits The size of the MAC key in bits. - * \param alg A MAC algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_MAC(\p alg) is true). - * - * \return The MAC size for the specified algorithm with - * the specified key parameters. - * \return 0 if the MAC algorithm is not recognized. - * \return Either 0 or the correct size for a MAC algorithm that - * the implementation recognizes, but does not support. - * \return Unspecified if the key parameters are not consistent - * with the algorithm. - */ -#define PSA_MAC_LENGTH(key_type, key_bits, alg) \ - ((alg) & PSA_ALG_MAC_TRUNCATION_MASK ? PSA_MAC_TRUNCATED_LENGTH(alg) : \ - PSA_ALG_IS_HMAC(alg) ? PSA_HASH_LENGTH(PSA_ALG_HMAC_GET_HASH(alg)) : \ - PSA_ALG_IS_BLOCK_CIPHER_MAC(alg) ? PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) : \ - ((void) (key_type), (void) (key_bits), 0u)) - -/** The maximum size of the output of psa_aead_encrypt(), in bytes. - * - * If the size of the ciphertext buffer is at least this large, it is - * guaranteed that psa_aead_encrypt() will not fail due to an - * insufficient buffer size. Depending on the algorithm, the actual size of - * the ciphertext may be smaller. - * - * See also #PSA_AEAD_ENCRYPT_OUTPUT_MAX_SIZE(\p plaintext_length). - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is - * compatible with algorithm \p alg. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * \param plaintext_length Size of the plaintext in bytes. - * - * \return The AEAD ciphertext size for the specified - * algorithm. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_AEAD_ENCRYPT_OUTPUT_SIZE(key_type, alg, plaintext_length) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 ? \ - (plaintext_length) + PSA_ALG_AEAD_GET_TAG_LENGTH(alg) : \ - 0u) - -/** A sufficient output buffer size for psa_aead_encrypt(), for any of the - * supported key types and AEAD algorithms. - * - * If the size of the ciphertext buffer is at least this large, it is guaranteed - * that psa_aead_encrypt() will not fail due to an insufficient buffer size. - * - * \note This macro returns a compile-time constant if its arguments are - * compile-time constants. - * - * See also #PSA_AEAD_ENCRYPT_OUTPUT_SIZE(\p key_type, \p alg, - * \p plaintext_length). - * - * \param plaintext_length Size of the plaintext in bytes. - * - * \return A sufficient output buffer size for any of the - * supported key types and AEAD algorithms. - * - */ -#define PSA_AEAD_ENCRYPT_OUTPUT_MAX_SIZE(plaintext_length) \ - ((plaintext_length) + PSA_AEAD_TAG_MAX_SIZE) - - -/** The maximum size of the output of psa_aead_decrypt(), in bytes. - * - * If the size of the plaintext buffer is at least this large, it is - * guaranteed that psa_aead_decrypt() will not fail due to an - * insufficient buffer size. Depending on the algorithm, the actual size of - * the plaintext may be smaller. - * - * See also #PSA_AEAD_DECRYPT_OUTPUT_MAX_SIZE(\p ciphertext_length). - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is - * compatible with algorithm \p alg. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * \param ciphertext_length Size of the plaintext in bytes. - * - * \return The AEAD ciphertext size for the specified - * algorithm. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_AEAD_DECRYPT_OUTPUT_SIZE(key_type, alg, ciphertext_length) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 && \ - (ciphertext_length) > PSA_ALG_AEAD_GET_TAG_LENGTH(alg) ? \ - (ciphertext_length) - PSA_ALG_AEAD_GET_TAG_LENGTH(alg) : \ - 0u) - -/** A sufficient output buffer size for psa_aead_decrypt(), for any of the - * supported key types and AEAD algorithms. - * - * If the size of the plaintext buffer is at least this large, it is guaranteed - * that psa_aead_decrypt() will not fail due to an insufficient buffer size. - * - * \note This macro returns a compile-time constant if its arguments are - * compile-time constants. - * - * See also #PSA_AEAD_DECRYPT_OUTPUT_SIZE(\p key_type, \p alg, - * \p ciphertext_length). - * - * \param ciphertext_length Size of the ciphertext in bytes. - * - * \return A sufficient output buffer size for any of the - * supported key types and AEAD algorithms. - * - */ -#define PSA_AEAD_DECRYPT_OUTPUT_MAX_SIZE(ciphertext_length) \ - (ciphertext_length) - -/** The default nonce size for an AEAD algorithm, in bytes. - * - * This macro can be used to allocate a buffer of sufficient size to - * store the nonce output from #psa_aead_generate_nonce(). - * - * See also #PSA_AEAD_NONCE_MAX_SIZE. - * - * \note This is not the maximum size of nonce supported as input to - * #psa_aead_set_nonce(), #psa_aead_encrypt() or #psa_aead_decrypt(), - * just the default size that is generated by #psa_aead_generate_nonce(). - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is compatible with - * algorithm \p alg. - * - * \param alg An AEAD algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \return The default nonce size for the specified key type and algorithm. - * If the key type or AEAD algorithm is not recognized, - * or the parameters are incompatible, return 0. - */ -#define PSA_AEAD_NONCE_LENGTH(key_type, alg) \ - (PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) == 16 ? \ - MBEDTLS_PSA_ALG_AEAD_EQUAL(alg, PSA_ALG_CCM) ? 13u : \ - MBEDTLS_PSA_ALG_AEAD_EQUAL(alg, PSA_ALG_GCM) ? 12u : \ - 0u : \ - (key_type) == PSA_KEY_TYPE_CHACHA20 && \ - MBEDTLS_PSA_ALG_AEAD_EQUAL(alg, PSA_ALG_CHACHA20_POLY1305) ? 12u : \ - 0u) - -/** The maximum default nonce size among all supported pairs of key types and - * AEAD algorithms, in bytes. - * - * This is equal to or greater than any value that #PSA_AEAD_NONCE_LENGTH() - * may return. - * - * \note This is not the maximum size of nonce supported as input to - * #psa_aead_set_nonce(), #psa_aead_encrypt() or #psa_aead_decrypt(), - * just the largest size that may be generated by - * #psa_aead_generate_nonce(). - */ -#define PSA_AEAD_NONCE_MAX_SIZE 13u - -/** A sufficient output buffer size for psa_aead_update(). - * - * If the size of the output buffer is at least this large, it is - * guaranteed that psa_aead_update() will not fail due to an - * insufficient buffer size. The actual size of the output may be smaller - * in any given call. - * - * See also #PSA_AEAD_UPDATE_OUTPUT_MAX_SIZE(\p input_length). - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is - * compatible with algorithm \p alg. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * \param input_length Size of the input in bytes. - * - * \return A sufficient output buffer size for the specified - * algorithm. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -/* For all the AEAD modes defined in this specification, it is possible - * to emit output without delay. However, hardware may not always be - * capable of this. So for modes based on a block cipher, allow the - * implementation to delay the output until it has a full block. */ -#define PSA_AEAD_UPDATE_OUTPUT_SIZE(key_type, alg, input_length) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 ? \ - PSA_ALG_IS_AEAD_ON_BLOCK_CIPHER(alg) ? \ - PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type), (input_length)) : \ - (input_length) : \ - 0u) - -/** A sufficient output buffer size for psa_aead_update(), for any of the - * supported key types and AEAD algorithms. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_aead_update() will not fail due to an insufficient buffer size. - * - * See also #PSA_AEAD_UPDATE_OUTPUT_SIZE(\p key_type, \p alg, \p input_length). - * - * \param input_length Size of the input in bytes. - */ -#define PSA_AEAD_UPDATE_OUTPUT_MAX_SIZE(input_length) \ - (PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE, (input_length))) - -/** A sufficient ciphertext buffer size for psa_aead_finish(). - * - * If the size of the ciphertext buffer is at least this large, it is - * guaranteed that psa_aead_finish() will not fail due to an - * insufficient ciphertext buffer size. The actual size of the output may - * be smaller in any given call. - * - * See also #PSA_AEAD_FINISH_OUTPUT_MAX_SIZE. - * - * \param key_type A symmetric key type that is - compatible with algorithm \p alg. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \return A sufficient ciphertext buffer size for the - * specified algorithm. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_AEAD_FINISH_OUTPUT_SIZE(key_type, alg) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 && \ - PSA_ALG_IS_AEAD_ON_BLOCK_CIPHER(alg) ? \ - PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) : \ - 0u) - -/** A sufficient ciphertext buffer size for psa_aead_finish(), for any of the - * supported key types and AEAD algorithms. - * - * See also #PSA_AEAD_FINISH_OUTPUT_SIZE(\p key_type, \p alg). - */ -#define PSA_AEAD_FINISH_OUTPUT_MAX_SIZE (PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE) - -/** A sufficient plaintext buffer size for psa_aead_verify(). - * - * If the size of the plaintext buffer is at least this large, it is - * guaranteed that psa_aead_verify() will not fail due to an - * insufficient plaintext buffer size. The actual size of the output may - * be smaller in any given call. - * - * See also #PSA_AEAD_VERIFY_OUTPUT_MAX_SIZE. - * - * \param key_type A symmetric key type that is - * compatible with algorithm \p alg. - * \param alg An AEAD algorithm - * (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p alg) is true). - * - * \return A sufficient plaintext buffer size for the - * specified algorithm. - * If the key type or AEAD algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_AEAD_VERIFY_OUTPUT_SIZE(key_type, alg) \ - (PSA_AEAD_NONCE_LENGTH(key_type, alg) != 0 && \ - PSA_ALG_IS_AEAD_ON_BLOCK_CIPHER(alg) ? \ - PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) : \ - 0u) - -/** A sufficient plaintext buffer size for psa_aead_verify(), for any of the - * supported key types and AEAD algorithms. - * - * See also #PSA_AEAD_VERIFY_OUTPUT_SIZE(\p key_type, \p alg). - */ -#define PSA_AEAD_VERIFY_OUTPUT_MAX_SIZE (PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE) - -#define PSA_RSA_MINIMUM_PADDING_SIZE(alg) \ - (PSA_ALG_IS_RSA_OAEP(alg) ? \ - 2u * PSA_HASH_LENGTH(PSA_ALG_RSA_OAEP_GET_HASH(alg)) + 1u : \ - 11u /*PKCS#1v1.5*/) - -/** - * \brief ECDSA signature size for a given curve bit size - * - * \param curve_bits Curve size in bits. - * \return Signature size in bytes. - * - * \note This macro returns a compile-time constant if its argument is one. - */ -#define PSA_ECDSA_SIGNATURE_SIZE(curve_bits) \ - (PSA_BITS_TO_BYTES(curve_bits) * 2u) - -/** Sufficient signature buffer size for psa_sign_hash(). - * - * This macro returns a sufficient buffer size for a signature using a key - * of the specified type and size, with the specified algorithm. - * Note that the actual size of the signature may be smaller - * (some algorithms produce a variable-size signature). - * - * \warning This function may call its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type An asymmetric key type (this may indifferently be a - * key pair type or a public key type). - * \param key_bits The size of the key in bits. - * \param alg The signature algorithm. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_sign_hash() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that is not supported, - * return either a sensible size or 0. - * If the parameters are not valid, the - * return value is unspecified. - */ -#define PSA_SIGN_OUTPUT_SIZE(key_type, key_bits, alg) \ - (PSA_KEY_TYPE_IS_RSA(key_type) ? ((void) alg, PSA_BITS_TO_BYTES(key_bits)) : \ - PSA_KEY_TYPE_IS_ECC(key_type) ? PSA_ECDSA_SIGNATURE_SIZE(key_bits) : \ - ((void) alg, 0u)) - -#define PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE \ - PSA_ECDSA_SIGNATURE_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) - -/** \def PSA_SIGNATURE_MAX_SIZE - * - * Maximum size of an asymmetric signature. - * - * This macro expands to a compile-time constant integer. This value - * is the maximum size of a signature in bytes. - */ -#define PSA_SIGNATURE_MAX_SIZE 1 - -#if (defined(PSA_WANT_ALG_ECDSA) || defined(PSA_WANT_ALG_DETERMINISTIC_ECDSA)) && \ - (PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE > PSA_SIGNATURE_MAX_SIZE) -#undef PSA_SIGNATURE_MAX_SIZE -#define PSA_SIGNATURE_MAX_SIZE PSA_VENDOR_ECDSA_SIGNATURE_MAX_SIZE -#endif -#if (defined(PSA_WANT_ALG_RSA_PKCS1V15_SIGN) || defined(PSA_WANT_ALG_RSA_PSS)) && \ - (PSA_BITS_TO_BYTES(PSA_VENDOR_RSA_MAX_KEY_BITS) > PSA_SIGNATURE_MAX_SIZE) -#undef PSA_SIGNATURE_MAX_SIZE -#define PSA_SIGNATURE_MAX_SIZE PSA_BITS_TO_BYTES(PSA_VENDOR_RSA_MAX_KEY_BITS) -#endif - -/** Sufficient output buffer size for psa_asymmetric_encrypt(). - * - * This macro returns a sufficient buffer size for a ciphertext produced using - * a key of the specified type and size, with the specified algorithm. - * Note that the actual size of the ciphertext may be smaller, depending - * on the algorithm. - * - * \warning This function may call its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type An asymmetric key type (this may indifferently be a - * key pair type or a public key type). - * \param key_bits The size of the key in bits. - * \param alg The asymmetric encryption algorithm. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_asymmetric_encrypt() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that is not supported, - * return either a sensible size or 0. - * If the parameters are not valid, the - * return value is unspecified. - */ -#define PSA_ASYMMETRIC_ENCRYPT_OUTPUT_SIZE(key_type, key_bits, alg) \ - (PSA_KEY_TYPE_IS_RSA(key_type) ? \ - ((void) alg, PSA_BITS_TO_BYTES(key_bits)) : \ - 0u) - -/** A sufficient output buffer size for psa_asymmetric_encrypt(), for any - * supported asymmetric encryption. - * - * See also #PSA_ASYMMETRIC_ENCRYPT_OUTPUT_SIZE(\p key_type, \p key_bits, \p alg). - */ -/* This macro assumes that RSA is the only supported asymmetric encryption. */ -#define PSA_ASYMMETRIC_ENCRYPT_OUTPUT_MAX_SIZE \ - (PSA_BITS_TO_BYTES(PSA_VENDOR_RSA_MAX_KEY_BITS)) - -/** Sufficient output buffer size for psa_asymmetric_decrypt(). - * - * This macro returns a sufficient buffer size for a plaintext produced using - * a key of the specified type and size, with the specified algorithm. - * Note that the actual size of the plaintext may be smaller, depending - * on the algorithm. - * - * \warning This function may call its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type An asymmetric key type (this may indifferently be a - * key pair type or a public key type). - * \param key_bits The size of the key in bits. - * \param alg The asymmetric encryption algorithm. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_asymmetric_decrypt() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that is not supported, - * return either a sensible size or 0. - * If the parameters are not valid, the - * return value is unspecified. - */ -#define PSA_ASYMMETRIC_DECRYPT_OUTPUT_SIZE(key_type, key_bits, alg) \ - (PSA_KEY_TYPE_IS_RSA(key_type) ? \ - PSA_BITS_TO_BYTES(key_bits) - PSA_RSA_MINIMUM_PADDING_SIZE(alg) : \ - 0u) - -/** A sufficient output buffer size for psa_asymmetric_decrypt(), for any - * supported asymmetric decryption. - * - * This macro assumes that RSA is the only supported asymmetric encryption. - * - * See also #PSA_ASYMMETRIC_DECRYPT_OUTPUT_SIZE(\p key_type, \p key_bits, \p alg). - */ -#define PSA_ASYMMETRIC_DECRYPT_OUTPUT_MAX_SIZE \ - (PSA_BITS_TO_BYTES(PSA_VENDOR_RSA_MAX_KEY_BITS)) - -/* Maximum size of the ASN.1 encoding of an INTEGER with the specified - * number of bits. - * - * This definition assumes that bits <= 2^19 - 9 so that the length field - * is at most 3 bytes. The length of the encoding is the length of the - * bit string padded to a whole number of bytes plus: - * - 1 type byte; - * - 1 to 3 length bytes; - * - 0 to 1 bytes of leading 0 due to the sign bit. - */ -#define PSA_KEY_EXPORT_ASN1_INTEGER_MAX_SIZE(bits) \ - ((bits) / 8u + 5u) - -/* Maximum size of the export encoding of an RSA public key. - * Assumes that the public exponent is less than 2^32. - * - * RSAPublicKey ::= SEQUENCE { - * modulus INTEGER, -- n - * publicExponent INTEGER } -- e - * - * - 4 bytes of SEQUENCE overhead; - * - n : INTEGER; - * - 7 bytes for the public exponent. - */ -#define PSA_KEY_EXPORT_RSA_PUBLIC_KEY_MAX_SIZE(key_bits) \ - (PSA_KEY_EXPORT_ASN1_INTEGER_MAX_SIZE(key_bits) + 11u) - -/* Maximum size of the export encoding of an RSA key pair. - * Assumes that the public exponent is less than 2^32 and that the size - * difference between the two primes is at most 1 bit. - * - * RSAPrivateKey ::= SEQUENCE { - * version Version, -- 0 - * modulus INTEGER, -- N-bit - * publicExponent INTEGER, -- 32-bit - * privateExponent INTEGER, -- N-bit - * prime1 INTEGER, -- N/2-bit - * prime2 INTEGER, -- N/2-bit - * exponent1 INTEGER, -- N/2-bit - * exponent2 INTEGER, -- N/2-bit - * coefficient INTEGER, -- N/2-bit - * } - * - * - 4 bytes of SEQUENCE overhead; - * - 3 bytes of version; - * - 7 half-size INTEGERs plus 2 full-size INTEGERs, - * overapproximated as 9 half-size INTEGERS; - * - 7 bytes for the public exponent. - */ -#define PSA_KEY_EXPORT_RSA_KEY_PAIR_MAX_SIZE(key_bits) \ - (9u * PSA_KEY_EXPORT_ASN1_INTEGER_MAX_SIZE((key_bits) / 2u + 1u) + 14u) - -/* Maximum size of the export encoding of a DSA public key. - * - * SubjectPublicKeyInfo ::= SEQUENCE { - * algorithm AlgorithmIdentifier, - * subjectPublicKey BIT STRING } -- contains DSAPublicKey - * AlgorithmIdentifier ::= SEQUENCE { - * algorithm OBJECT IDENTIFIER, - * parameters Dss-Params } -- SEQUENCE of 3 INTEGERs - * DSAPublicKey ::= INTEGER -- public key, Y - * - * - 3 * 4 bytes of SEQUENCE overhead; - * - 1 + 1 + 7 bytes of algorithm (DSA OID); - * - 4 bytes of BIT STRING overhead; - * - 3 full-size INTEGERs (p, g, y); - * - 1 + 1 + 32 bytes for 1 sub-size INTEGER (q <= 256 bits). - */ -#define PSA_KEY_EXPORT_DSA_PUBLIC_KEY_MAX_SIZE(key_bits) \ - (PSA_KEY_EXPORT_ASN1_INTEGER_MAX_SIZE(key_bits) * 3u + 59u) - -/* Maximum size of the export encoding of a DSA key pair. - * - * DSAPrivateKey ::= SEQUENCE { - * version Version, -- 0 - * prime INTEGER, -- p - * subprime INTEGER, -- q - * generator INTEGER, -- g - * public INTEGER, -- y - * private INTEGER, -- x - * } - * - * - 4 bytes of SEQUENCE overhead; - * - 3 bytes of version; - * - 3 full-size INTEGERs (p, g, y); - * - 2 * (1 + 1 + 32) bytes for 2 sub-size INTEGERs (q, x <= 256 bits). - */ -#define PSA_KEY_EXPORT_DSA_KEY_PAIR_MAX_SIZE(key_bits) \ - (PSA_KEY_EXPORT_ASN1_INTEGER_MAX_SIZE(key_bits) * 3u + 75u) - -/* Maximum size of the export encoding of an ECC public key. - * - * The representation of an ECC public key is: - * - The byte 0x04; - * - `x_P` as a `ceiling(m/8)`-byte string, big-endian; - * - `y_P` as a `ceiling(m/8)`-byte string, big-endian; - * - where m is the bit size associated with the curve. - * - * - 1 byte + 2 * point size. - */ -#define PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(key_bits) \ - (2u * PSA_BITS_TO_BYTES(key_bits) + 1u) - -/* Maximum size of the export encoding of an ECC key pair. - * - * An ECC key pair is represented by the secret value. - */ -#define PSA_KEY_EXPORT_ECC_KEY_PAIR_MAX_SIZE(key_bits) \ - (PSA_BITS_TO_BYTES(key_bits)) - -/* Maximum size of the export encoding of an DH key pair. - * - * An DH key pair is represented by the secret value. - */ -#define PSA_KEY_EXPORT_FFDH_KEY_PAIR_MAX_SIZE(key_bits) \ - (PSA_BITS_TO_BYTES(key_bits)) - -/* Maximum size of the export encoding of an DH public key. - */ -#define PSA_KEY_EXPORT_FFDH_PUBLIC_KEY_MAX_SIZE(key_bits) \ - (PSA_BITS_TO_BYTES(key_bits)) - -/** Sufficient output buffer size for psa_export_key() or - * psa_export_public_key(). - * - * This macro returns a compile-time constant if its arguments are - * compile-time constants. - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * The following code illustrates how to allocate enough memory to export - * a key by querying the key type and size at runtime. - * \code{c} - * psa_key_attributes_t attributes = PSA_KEY_ATTRIBUTES_INIT; - * psa_status_t status; - * status = psa_get_key_attributes(key, &attributes); - * if (status != PSA_SUCCESS) handle_error(...); - * psa_key_type_t key_type = psa_get_key_type(&attributes); - * size_t key_bits = psa_get_key_bits(&attributes); - * size_t buffer_size = PSA_EXPORT_KEY_OUTPUT_SIZE(key_type, key_bits); - * psa_reset_key_attributes(&attributes); - * uint8_t *buffer = malloc(buffer_size); - * if (buffer == NULL) handle_error(...); - * size_t buffer_length; - * status = psa_export_key(key, buffer, buffer_size, &buffer_length); - * if (status != PSA_SUCCESS) handle_error(...); - * \endcode - * - * \param key_type A supported key type. - * \param key_bits The size of the key in bits. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_export_key() or psa_export_public_key() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that is not supported, - * return either a sensible size or 0. - * If the parameters are not valid, the return value is unspecified. - */ -#define PSA_EXPORT_KEY_OUTPUT_SIZE(key_type, key_bits) \ - (PSA_KEY_TYPE_IS_UNSTRUCTURED(key_type) ? PSA_BITS_TO_BYTES(key_bits) : \ - PSA_KEY_TYPE_IS_DH(key_type) ? PSA_BITS_TO_BYTES(key_bits) : \ - (key_type) == PSA_KEY_TYPE_RSA_KEY_PAIR ? PSA_KEY_EXPORT_RSA_KEY_PAIR_MAX_SIZE(key_bits) : \ - (key_type) == PSA_KEY_TYPE_RSA_PUBLIC_KEY ? PSA_KEY_EXPORT_RSA_PUBLIC_KEY_MAX_SIZE(key_bits) : \ - (key_type) == PSA_KEY_TYPE_DSA_KEY_PAIR ? PSA_KEY_EXPORT_DSA_KEY_PAIR_MAX_SIZE(key_bits) : \ - (key_type) == PSA_KEY_TYPE_DSA_PUBLIC_KEY ? PSA_KEY_EXPORT_DSA_PUBLIC_KEY_MAX_SIZE(key_bits) : \ - PSA_KEY_TYPE_IS_ECC_KEY_PAIR(key_type) ? PSA_KEY_EXPORT_ECC_KEY_PAIR_MAX_SIZE(key_bits) : \ - PSA_KEY_TYPE_IS_ECC_PUBLIC_KEY(key_type) ? PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(key_bits) : \ - 0u) - -/** Sufficient output buffer size for psa_export_public_key(). - * - * This macro returns a compile-time constant if its arguments are - * compile-time constants. - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * The following code illustrates how to allocate enough memory to export - * a public key by querying the key type and size at runtime. - * \code{c} - * psa_key_attributes_t attributes = PSA_KEY_ATTRIBUTES_INIT; - * psa_status_t status; - * status = psa_get_key_attributes(key, &attributes); - * if (status != PSA_SUCCESS) handle_error(...); - * psa_key_type_t key_type = psa_get_key_type(&attributes); - * size_t key_bits = psa_get_key_bits(&attributes); - * size_t buffer_size = PSA_EXPORT_PUBLIC_KEY_OUTPUT_SIZE(key_type, key_bits); - * psa_reset_key_attributes(&attributes); - * uint8_t *buffer = malloc(buffer_size); - * if (buffer == NULL) handle_error(...); - * size_t buffer_length; - * status = psa_export_public_key(key, buffer, buffer_size, &buffer_length); - * if (status != PSA_SUCCESS) handle_error(...); - * \endcode - * - * \param key_type A public key or key pair key type. - * \param key_bits The size of the key in bits. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_export_public_key() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that is not - * supported, return either a sensible size or 0. - * If the parameters are not valid, - * the return value is unspecified. - * - * If the parameters are valid and supported, - * return the same result as - * #PSA_EXPORT_KEY_OUTPUT_SIZE( - * \p #PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(\p key_type), - * \p key_bits). - */ -#define PSA_EXPORT_PUBLIC_KEY_OUTPUT_SIZE(key_type, key_bits) \ - (PSA_KEY_TYPE_IS_RSA(key_type) ? PSA_KEY_EXPORT_RSA_PUBLIC_KEY_MAX_SIZE(key_bits) : \ - PSA_KEY_TYPE_IS_ECC(key_type) ? PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(key_bits) : \ - PSA_KEY_TYPE_IS_DH(key_type) ? PSA_BITS_TO_BYTES(key_bits) : \ - 0u) - -/** Sufficient buffer size for exporting any asymmetric key pair. - * - * This macro expands to a compile-time constant integer. This value is - * a sufficient buffer size when calling psa_export_key() to export any - * asymmetric key pair, regardless of the exact key type and key size. - * - * See also #PSA_EXPORT_KEY_OUTPUT_SIZE(\p key_type, \p key_bits). - */ -#define PSA_EXPORT_KEY_PAIR_MAX_SIZE 1 - -#if defined(PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_BASIC) && \ - (PSA_KEY_EXPORT_ECC_KEY_PAIR_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) > \ - PSA_EXPORT_KEY_PAIR_MAX_SIZE) -#undef PSA_EXPORT_KEY_PAIR_MAX_SIZE -#define PSA_EXPORT_KEY_PAIR_MAX_SIZE \ - PSA_KEY_EXPORT_ECC_KEY_PAIR_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) -#endif -#if defined(PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_BASIC) && \ - (PSA_KEY_EXPORT_RSA_KEY_PAIR_MAX_SIZE(PSA_VENDOR_RSA_MAX_KEY_BITS) > \ - PSA_EXPORT_KEY_PAIR_MAX_SIZE) -#undef PSA_EXPORT_KEY_PAIR_MAX_SIZE -#define PSA_EXPORT_KEY_PAIR_MAX_SIZE \ - PSA_KEY_EXPORT_RSA_KEY_PAIR_MAX_SIZE(PSA_VENDOR_RSA_MAX_KEY_BITS) -#endif -#if defined(PSA_WANT_KEY_TYPE_DH_KEY_PAIR_BASIC) && \ - (PSA_KEY_EXPORT_FFDH_KEY_PAIR_MAX_SIZE(PSA_VENDOR_FFDH_MAX_KEY_BITS) > \ - PSA_EXPORT_KEY_PAIR_MAX_SIZE) -#undef PSA_EXPORT_KEY_PAIR_MAX_SIZE -#define PSA_EXPORT_KEY_PAIR_MAX_SIZE \ - PSA_KEY_EXPORT_FFDH_KEY_PAIR_MAX_SIZE(PSA_VENDOR_FFDH_MAX_KEY_BITS) -#endif - -/** Sufficient buffer size for exporting any asymmetric public key. - * - * This macro expands to a compile-time constant integer. This value is - * a sufficient buffer size when calling psa_export_key() or - * psa_export_public_key() to export any asymmetric public key, - * regardless of the exact key type and key size. - * - * See also #PSA_EXPORT_PUBLIC_KEY_OUTPUT_SIZE(\p key_type, \p key_bits). - */ -#define PSA_EXPORT_PUBLIC_KEY_MAX_SIZE 1 - -#if defined(PSA_WANT_KEY_TYPE_ECC_PUBLIC_KEY) && \ - (PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) > \ - PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) -#undef PSA_EXPORT_PUBLIC_KEY_MAX_SIZE -#define PSA_EXPORT_PUBLIC_KEY_MAX_SIZE \ - PSA_KEY_EXPORT_ECC_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_ECC_MAX_CURVE_BITS) -#endif -#if defined(PSA_WANT_KEY_TYPE_RSA_PUBLIC_KEY) && \ - (PSA_KEY_EXPORT_RSA_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_RSA_MAX_KEY_BITS) > \ - PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) -#undef PSA_EXPORT_PUBLIC_KEY_MAX_SIZE -#define PSA_EXPORT_PUBLIC_KEY_MAX_SIZE \ - PSA_KEY_EXPORT_RSA_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_RSA_MAX_KEY_BITS) -#endif -#if defined(PSA_WANT_KEY_TYPE_DH_PUBLIC_KEY) && \ - (PSA_KEY_EXPORT_FFDH_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_FFDH_MAX_KEY_BITS) > \ - PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) -#undef PSA_EXPORT_PUBLIC_KEY_MAX_SIZE -#define PSA_EXPORT_PUBLIC_KEY_MAX_SIZE \ - PSA_KEY_EXPORT_FFDH_PUBLIC_KEY_MAX_SIZE(PSA_VENDOR_FFDH_MAX_KEY_BITS) -#endif - -#define PSA_EXPORT_KEY_PAIR_OR_PUBLIC_MAX_SIZE \ - ((PSA_EXPORT_KEY_PAIR_MAX_SIZE > PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) ? \ - PSA_EXPORT_KEY_PAIR_MAX_SIZE : PSA_EXPORT_PUBLIC_KEY_MAX_SIZE) - -/** Sufficient output buffer size for psa_raw_key_agreement(). - * - * This macro returns a compile-time constant if its arguments are - * compile-time constants. - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * See also #PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE. - * - * \param key_type A supported key type. - * \param key_bits The size of the key in bits. - * - * \return If the parameters are valid and supported, return - * a buffer size in bytes that guarantees that - * psa_raw_key_agreement() will not fail with - * #PSA_ERROR_BUFFER_TOO_SMALL. - * If the parameters are a valid combination that - * is not supported, return either a sensible size or 0. - * If the parameters are not valid, - * the return value is unspecified. - */ -#define PSA_RAW_KEY_AGREEMENT_OUTPUT_SIZE(key_type, key_bits) \ - ((PSA_KEY_TYPE_IS_ECC_KEY_PAIR(key_type) || \ - PSA_KEY_TYPE_IS_DH_KEY_PAIR(key_type)) ? PSA_BITS_TO_BYTES(key_bits) : 0u) - -/** Maximum size of the output from psa_raw_key_agreement(). - * - * This macro expands to a compile-time constant integer. This value is the - * maximum size of the output any raw key agreement algorithm, in bytes. - * - * See also #PSA_RAW_KEY_AGREEMENT_OUTPUT_SIZE(\p key_type, \p key_bits). - */ -#define PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE 1 - -#if defined(PSA_WANT_ALG_ECDH) && \ - (PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS) > PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE) -#undef PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE -#define PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE PSA_BITS_TO_BYTES(PSA_VENDOR_ECC_MAX_CURVE_BITS) -#endif -#if defined(PSA_WANT_ALG_FFDH) && \ - (PSA_BITS_TO_BYTES(PSA_VENDOR_FFDH_MAX_KEY_BITS) > PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE) -#undef PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE -#define PSA_RAW_KEY_AGREEMENT_OUTPUT_MAX_SIZE PSA_BITS_TO_BYTES(PSA_VENDOR_FFDH_MAX_KEY_BITS) -#endif - -/** Maximum key length for ciphers. - * - * Since there is no additional PSA_WANT_xxx symbol to specifiy the size of - * the key once a cipher is enabled (as it happens for asymmetric keys for - * example), the maximum key length is taken into account for each cipher. - * The resulting value will be the maximum cipher's key length given depending - * on which ciphers are enabled. - * - * Note: max value for AES used below would be doubled if XTS were enabled, but - * this mode is currently not supported in Mbed TLS implementation of PSA - * APIs. - */ -#if (defined(PSA_WANT_KEY_TYPE_AES) || defined(PSA_WANT_KEY_TYPE_ARIA) || \ - defined(PSA_WANT_KEY_TYPE_CAMELLIA) || defined(PSA_WANT_KEY_TYPE_CHACHA20)) -#define PSA_CIPHER_MAX_KEY_LENGTH 32u -#elif defined(PSA_WANT_KEY_TYPE_DES) -#define PSA_CIPHER_MAX_KEY_LENGTH 24u -#else -#define PSA_CIPHER_MAX_KEY_LENGTH 0u -#endif - -/** The default IV size for a cipher algorithm, in bytes. - * - * The IV that is generated as part of a call to #psa_cipher_encrypt() is always - * the default IV length for the algorithm. - * - * This macro can be used to allocate a buffer of sufficient size to - * store the IV output from #psa_cipher_generate_iv() when using - * a multi-part cipher operation. - * - * See also #PSA_CIPHER_IV_MAX_SIZE. - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is compatible with algorithm \p alg. - * - * \param alg A cipher algorithm (\c PSA_ALG_XXX value such that #PSA_ALG_IS_CIPHER(\p alg) is true). - * - * \return The default IV size for the specified key type and algorithm. - * If the algorithm does not use an IV, return 0. - * If the key type or cipher algorithm is not recognized, - * or the parameters are incompatible, return 0. - */ -#define PSA_CIPHER_IV_LENGTH(key_type, alg) \ - (PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) > 1 && \ - ((alg) == PSA_ALG_CTR || \ - (alg) == PSA_ALG_CFB || \ - (alg) == PSA_ALG_OFB || \ - (alg) == PSA_ALG_XTS || \ - (alg) == PSA_ALG_CBC_NO_PADDING || \ - (alg) == PSA_ALG_CBC_PKCS7) ? PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) : \ - (key_type) == PSA_KEY_TYPE_CHACHA20 && \ - (alg) == PSA_ALG_STREAM_CIPHER ? 12u : \ - (alg) == PSA_ALG_CCM_STAR_NO_TAG ? 13u : \ - 0u) - -/** The maximum IV size for all supported cipher algorithms, in bytes. - * - * See also #PSA_CIPHER_IV_LENGTH(). - */ -#define PSA_CIPHER_IV_MAX_SIZE 16u - -/** The maximum size of the output of psa_cipher_encrypt(), in bytes. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_encrypt() will not fail due to an insufficient buffer size. - * Depending on the algorithm, the actual size of the output might be smaller. - * - * See also #PSA_CIPHER_ENCRYPT_OUTPUT_MAX_SIZE(\p input_length). - * - * \warning This macro may evaluate its arguments multiple times or - * zero times, so you should not pass arguments that contain - * side effects. - * - * \param key_type A symmetric key type that is compatible with algorithm - * alg. - * \param alg A cipher algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \param input_length Size of the input in bytes. - * - * \return A sufficient output size for the specified key type and - * algorithm. If the key type or cipher algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_CIPHER_ENCRYPT_OUTPUT_SIZE(key_type, alg, input_length) \ - (alg == PSA_ALG_CBC_PKCS7 ? \ - (PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) != 0 ? \ - PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type), \ - (input_length) + 1u) + \ - PSA_CIPHER_IV_LENGTH((key_type), (alg)) : 0u) : \ - (PSA_ALG_IS_CIPHER(alg) ? \ - (input_length) + PSA_CIPHER_IV_LENGTH((key_type), (alg)) : \ - 0u)) - -/** A sufficient output buffer size for psa_cipher_encrypt(), for any of the - * supported key types and cipher algorithms. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_encrypt() will not fail due to an insufficient buffer size. - * - * See also #PSA_CIPHER_ENCRYPT_OUTPUT_SIZE(\p key_type, \p alg, \p input_length). - * - * \param input_length Size of the input in bytes. - * - */ -#define PSA_CIPHER_ENCRYPT_OUTPUT_MAX_SIZE(input_length) \ - (PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE, \ - (input_length) + 1u) + \ - PSA_CIPHER_IV_MAX_SIZE) - -/** The maximum size of the output of psa_cipher_decrypt(), in bytes. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_decrypt() will not fail due to an insufficient buffer size. - * Depending on the algorithm, the actual size of the output might be smaller. - * - * See also #PSA_CIPHER_DECRYPT_OUTPUT_MAX_SIZE(\p input_length). - * - * \param key_type A symmetric key type that is compatible with algorithm - * alg. - * \param alg A cipher algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \param input_length Size of the input in bytes. - * - * \return A sufficient output size for the specified key type and - * algorithm. If the key type or cipher algorithm is not - * recognized, or the parameters are incompatible, - * return 0. - */ -#define PSA_CIPHER_DECRYPT_OUTPUT_SIZE(key_type, alg, input_length) \ - (PSA_ALG_IS_CIPHER(alg) && \ - ((key_type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_SYMMETRIC ? \ - (input_length) : \ - 0u) - -/** A sufficient output buffer size for psa_cipher_decrypt(), for any of the - * supported key types and cipher algorithms. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_decrypt() will not fail due to an insufficient buffer size. - * - * See also #PSA_CIPHER_DECRYPT_OUTPUT_SIZE(\p key_type, \p alg, \p input_length). - * - * \param input_length Size of the input in bytes. - */ -#define PSA_CIPHER_DECRYPT_OUTPUT_MAX_SIZE(input_length) \ - (input_length) - -/** A sufficient output buffer size for psa_cipher_update(). - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_update() will not fail due to an insufficient buffer size. - * The actual size of the output might be smaller in any given call. - * - * See also #PSA_CIPHER_UPDATE_OUTPUT_MAX_SIZE(\p input_length). - * - * \param key_type A symmetric key type that is compatible with algorithm - * alg. - * \param alg A cipher algorithm (PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \param input_length Size of the input in bytes. - * - * \return A sufficient output size for the specified key type and - * algorithm. If the key type or cipher algorithm is not - * recognized, or the parameters are incompatible, return 0. - */ -#define PSA_CIPHER_UPDATE_OUTPUT_SIZE(key_type, alg, input_length) \ - (PSA_ALG_IS_CIPHER(alg) ? \ - (PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) != 0 ? \ - (((alg) == PSA_ALG_CBC_PKCS7 || \ - (alg) == PSA_ALG_CBC_NO_PADDING || \ - (alg) == PSA_ALG_ECB_NO_PADDING) ? \ - PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type), \ - input_length) : \ - (input_length)) : 0u) : \ - 0u) - -/** A sufficient output buffer size for psa_cipher_update(), for any of the - * supported key types and cipher algorithms. - * - * If the size of the output buffer is at least this large, it is guaranteed - * that psa_cipher_update() will not fail due to an insufficient buffer size. - * - * See also #PSA_CIPHER_UPDATE_OUTPUT_SIZE(\p key_type, \p alg, \p input_length). - * - * \param input_length Size of the input in bytes. - */ -#define PSA_CIPHER_UPDATE_OUTPUT_MAX_SIZE(input_length) \ - (PSA_ROUND_UP_TO_MULTIPLE(PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE, input_length)) - -/** A sufficient ciphertext buffer size for psa_cipher_finish(). - * - * If the size of the ciphertext buffer is at least this large, it is - * guaranteed that psa_cipher_finish() will not fail due to an insufficient - * ciphertext buffer size. The actual size of the output might be smaller in - * any given call. - * - * See also #PSA_CIPHER_FINISH_OUTPUT_MAX_SIZE(). - * - * \param key_type A symmetric key type that is compatible with algorithm - * alg. - * \param alg A cipher algorithm (PSA_ALG_XXX value such that - * #PSA_ALG_IS_CIPHER(\p alg) is true). - * \return A sufficient output size for the specified key type and - * algorithm. If the key type or cipher algorithm is not - * recognized, or the parameters are incompatible, return 0. - */ -#define PSA_CIPHER_FINISH_OUTPUT_SIZE(key_type, alg) \ - (PSA_ALG_IS_CIPHER(alg) ? \ - (alg == PSA_ALG_CBC_PKCS7 ? \ - PSA_BLOCK_CIPHER_BLOCK_LENGTH(key_type) : \ - 0u) : \ - 0u) - -/** A sufficient ciphertext buffer size for psa_cipher_finish(), for any of the - * supported key types and cipher algorithms. - * - * See also #PSA_CIPHER_FINISH_OUTPUT_SIZE(\p key_type, \p alg). - */ -#define PSA_CIPHER_FINISH_OUTPUT_MAX_SIZE \ - (PSA_BLOCK_CIPHER_BLOCK_MAX_SIZE) - -#endif /* PSA_CRYPTO_SIZES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_struct.h b/android/mbedtls/arm64-v8a/include/psa/crypto_struct.h deleted file mode 100644 index 362e921a3..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_struct.h +++ /dev/null @@ -1,527 +0,0 @@ -/** - * \file psa/crypto_struct.h - * - * \brief PSA cryptography module: Mbed TLS structured type implementations - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. - * - * This file contains the definitions of some data structures with - * implementation-specific definitions. - * - * In implementations with isolation between the application and the - * cryptography module, it is expected that the front-end and the back-end - * would have different versions of this file. - * - *

Design notes about multipart operation structures

- * - * For multipart operations without driver delegation support, each multipart - * operation structure contains a `psa_algorithm_t alg` field which indicates - * which specific algorithm the structure is for. When the structure is not in - * use, `alg` is 0. Most of the structure consists of a union which is - * discriminated by `alg`. - * - * For multipart operations with driver delegation support, each multipart - * operation structure contains an `unsigned int id` field indicating which - * driver got assigned to do the operation. When the structure is not in use, - * 'id' is 0. The structure contains also a driver context which is the union - * of the contexts of all drivers able to handle the type of multipart - * operation. - * - * Note that when `alg` or `id` is 0, the content of other fields is undefined. - * In particular, it is not guaranteed that a freshly-initialized structure - * is all-zero: we initialize structures to something like `{0, 0}`, which - * is only guaranteed to initializes the first member of the union; - * GCC and Clang initialize the whole structure to 0 (at the time of writing), - * but MSVC and CompCert don't. - * - * In Mbed TLS, multipart operation structures live independently from - * the key. This allows Mbed TLS to free the key objects when destroying - * a key slot. If a multipart operation needs to remember the key after - * the setup function returns, the operation structure needs to contain a - * copy of the key. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_STRUCT_H -#define PSA_CRYPTO_STRUCT_H -#include "mbedtls/private_access.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Include the build-time configuration information header. Here, we do not - * include `"mbedtls/build_info.h"` directly but `"psa/build_info.h"`, which - * is basically just an alias to it. This is to ease the maintenance of the - * TF-PSA-Crypto repository which has a different build system and - * configuration. - */ -#include "psa/build_info.h" - -/* Include the context definition for the compiled-in drivers for the primitive - * algorithms. */ -#include "psa/crypto_driver_contexts_primitives.h" - -struct psa_hash_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_driver_wrappers.h. - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. the driver context is not active, in use). */ - unsigned int MBEDTLS_PRIVATE(id); - psa_driver_hash_context_t MBEDTLS_PRIVATE(ctx); -#endif -}; -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_HASH_OPERATION_INIT { 0 } -#else -#define PSA_HASH_OPERATION_INIT { 0, { 0 } } -#endif -static inline struct psa_hash_operation_s psa_hash_operation_init(void) -{ - const struct psa_hash_operation_s v = PSA_HASH_OPERATION_INIT; - return v; -} - -struct psa_cipher_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_crypto_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - - unsigned int MBEDTLS_PRIVATE(iv_required) : 1; - unsigned int MBEDTLS_PRIVATE(iv_set) : 1; - - uint8_t MBEDTLS_PRIVATE(default_iv_length); - - psa_driver_cipher_context_t MBEDTLS_PRIVATE(ctx); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_CIPHER_OPERATION_INIT { 0 } -#else -#define PSA_CIPHER_OPERATION_INIT { 0, 0, 0, 0, { 0 } } -#endif -static inline struct psa_cipher_operation_s psa_cipher_operation_init(void) -{ - const struct psa_cipher_operation_s v = PSA_CIPHER_OPERATION_INIT; - return v; -} - -/* Include the context definition for the compiled-in drivers for the composite - * algorithms. */ -#include "psa/crypto_driver_contexts_composites.h" - -struct psa_mac_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - uint8_t MBEDTLS_PRIVATE(mac_size); - unsigned int MBEDTLS_PRIVATE(is_sign) : 1; - psa_driver_mac_context_t MBEDTLS_PRIVATE(ctx); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_MAC_OPERATION_INIT { 0 } -#else -#define PSA_MAC_OPERATION_INIT { 0, 0, 0, { 0 } } -#endif -static inline struct psa_mac_operation_s psa_mac_operation_init(void) -{ - const struct psa_mac_operation_s v = PSA_MAC_OPERATION_INIT; - return v; -} - -struct psa_aead_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_crypto_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - - psa_algorithm_t MBEDTLS_PRIVATE(alg); - psa_key_type_t MBEDTLS_PRIVATE(key_type); - - size_t MBEDTLS_PRIVATE(ad_remaining); - size_t MBEDTLS_PRIVATE(body_remaining); - - unsigned int MBEDTLS_PRIVATE(nonce_set) : 1; - unsigned int MBEDTLS_PRIVATE(lengths_set) : 1; - unsigned int MBEDTLS_PRIVATE(ad_started) : 1; - unsigned int MBEDTLS_PRIVATE(body_started) : 1; - unsigned int MBEDTLS_PRIVATE(is_encrypt) : 1; - - psa_driver_aead_context_t MBEDTLS_PRIVATE(ctx); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_AEAD_OPERATION_INIT { 0 } -#else -#define PSA_AEAD_OPERATION_INIT { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, { 0 } } -#endif -static inline struct psa_aead_operation_s psa_aead_operation_init(void) -{ - const struct psa_aead_operation_s v = PSA_AEAD_OPERATION_INIT; - return v; -} - -/* Include the context definition for the compiled-in drivers for the key - * derivation algorithms. */ -#include "psa/crypto_driver_contexts_key_derivation.h" - -struct psa_key_derivation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - psa_algorithm_t MBEDTLS_PRIVATE(alg); - unsigned int MBEDTLS_PRIVATE(can_output_key) : 1; - size_t MBEDTLS_PRIVATE(capacity); - psa_driver_key_derivation_context_t MBEDTLS_PRIVATE(ctx); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_KEY_DERIVATION_OPERATION_INIT { 0 } -#else -/* This only zeroes out the first byte in the union, the rest is unspecified. */ -#define PSA_KEY_DERIVATION_OPERATION_INIT { 0, 0, 0, { 0 } } -#endif -static inline struct psa_key_derivation_s psa_key_derivation_operation_init( - void) -{ - const struct psa_key_derivation_s v = PSA_KEY_DERIVATION_OPERATION_INIT; - return v; -} - -struct psa_custom_key_parameters_s { - /* Future versions may add other fields in this structure. */ - uint32_t flags; -}; - -/** The default production parameters for key generation or key derivation. - * - * Calling psa_generate_key_custom() or psa_key_derivation_output_key_custom() - * with `custom=PSA_CUSTOM_KEY_PARAMETERS_INIT` and `custom_data_length=0` is - * equivalent to calling psa_generate_key() or psa_key_derivation_output_key() - * respectively. - */ -#define PSA_CUSTOM_KEY_PARAMETERS_INIT { 0 } - -#ifndef __cplusplus -/* Omitted when compiling in C++, because one of the parameters is a - * pointer to a struct with a flexible array member, and that is not - * standard C++. - * https://github.com/Mbed-TLS/mbedtls/issues/9020 - */ -/* This is a deprecated variant of `struct psa_custom_key_parameters_s`. - * It has exactly the same layout, plus an extra field which is a flexible - * array member. Thus a `const struct psa_key_production_parameters_s *` - * can be passed to any function that reads a - * `const struct psa_custom_key_parameters_s *`. - */ -struct psa_key_production_parameters_s { - uint32_t flags; - uint8_t data[]; -}; - -/** The default production parameters for key generation or key derivation. - * - * Calling psa_generate_key_ext() or psa_key_derivation_output_key_ext() - * with `params=PSA_KEY_PRODUCTION_PARAMETERS_INIT` and - * `params_data_length == 0` is equivalent to - * calling psa_generate_key() or psa_key_derivation_output_key() - * respectively. - */ -#define PSA_KEY_PRODUCTION_PARAMETERS_INIT { 0 } -#endif /* !__cplusplus */ - -struct psa_key_policy_s { - psa_key_usage_t MBEDTLS_PRIVATE(usage); - psa_algorithm_t MBEDTLS_PRIVATE(alg); - psa_algorithm_t MBEDTLS_PRIVATE(alg2); -}; -typedef struct psa_key_policy_s psa_key_policy_t; - -#define PSA_KEY_POLICY_INIT { 0, 0, 0 } -static inline struct psa_key_policy_s psa_key_policy_init(void) -{ - const struct psa_key_policy_s v = PSA_KEY_POLICY_INIT; - return v; -} - -/* The type used internally for key sizes. - * Public interfaces use size_t, but internally we use a smaller type. */ -typedef uint16_t psa_key_bits_t; -/* The maximum value of the type used to represent bit-sizes. - * This is used to mark an invalid key size. */ -#define PSA_KEY_BITS_TOO_LARGE ((psa_key_bits_t) -1) -/* The maximum size of a key in bits. - * Currently defined as the maximum that can be represented, rounded down - * to a whole number of bytes. - * This is an uncast value so that it can be used in preprocessor - * conditionals. */ -#define PSA_MAX_KEY_BITS 0xfff8 - -struct psa_key_attributes_s { -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) - psa_key_slot_number_t MBEDTLS_PRIVATE(slot_number); - int MBEDTLS_PRIVATE(has_slot_number); -#endif /* MBEDTLS_PSA_CRYPTO_SE_C */ - psa_key_type_t MBEDTLS_PRIVATE(type); - psa_key_bits_t MBEDTLS_PRIVATE(bits); - psa_key_lifetime_t MBEDTLS_PRIVATE(lifetime); - psa_key_policy_t MBEDTLS_PRIVATE(policy); - /* This type has a different layout in the client view wrt the - * service view of the key id, i.e. in service view usually is - * expected to have MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER defined - * thus adding an owner field to the standard psa_key_id_t. For - * implementations with client/service separation, this means the - * object will be marshalled through a transport channel and - * interpreted differently at each side of the transport. Placing - * it at the end of structures allows to interpret the structure - * at the client without reorganizing the memory layout of the - * struct - */ - mbedtls_svc_key_id_t MBEDTLS_PRIVATE(id); -}; - -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) -#define PSA_KEY_ATTRIBUTES_MAYBE_SLOT_NUMBER 0, 0, -#else -#define PSA_KEY_ATTRIBUTES_MAYBE_SLOT_NUMBER -#endif -#define PSA_KEY_ATTRIBUTES_INIT { PSA_KEY_ATTRIBUTES_MAYBE_SLOT_NUMBER \ - PSA_KEY_TYPE_NONE, 0, \ - PSA_KEY_LIFETIME_VOLATILE, \ - PSA_KEY_POLICY_INIT, \ - MBEDTLS_SVC_KEY_ID_INIT } - -static inline struct psa_key_attributes_s psa_key_attributes_init(void) -{ - const struct psa_key_attributes_s v = PSA_KEY_ATTRIBUTES_INIT; - return v; -} - -static inline void psa_set_key_id(psa_key_attributes_t *attributes, - mbedtls_svc_key_id_t key) -{ - psa_key_lifetime_t lifetime = attributes->MBEDTLS_PRIVATE(lifetime); - - attributes->MBEDTLS_PRIVATE(id) = key; - - if (PSA_KEY_LIFETIME_IS_VOLATILE(lifetime)) { - attributes->MBEDTLS_PRIVATE(lifetime) = - PSA_KEY_LIFETIME_FROM_PERSISTENCE_AND_LOCATION( - PSA_KEY_LIFETIME_PERSISTENT, - PSA_KEY_LIFETIME_GET_LOCATION(lifetime)); - } -} - -static inline mbedtls_svc_key_id_t psa_get_key_id( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(id); -} - -#ifdef MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER -static inline void mbedtls_set_key_owner_id(psa_key_attributes_t *attributes, - mbedtls_key_owner_id_t owner) -{ - attributes->MBEDTLS_PRIVATE(id).MBEDTLS_PRIVATE(owner) = owner; -} -#endif - -static inline void psa_set_key_lifetime(psa_key_attributes_t *attributes, - psa_key_lifetime_t lifetime) -{ - attributes->MBEDTLS_PRIVATE(lifetime) = lifetime; - if (PSA_KEY_LIFETIME_IS_VOLATILE(lifetime)) { -#ifdef MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER - attributes->MBEDTLS_PRIVATE(id).MBEDTLS_PRIVATE(key_id) = 0; -#else - attributes->MBEDTLS_PRIVATE(id) = 0; -#endif - } -} - -static inline psa_key_lifetime_t psa_get_key_lifetime( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(lifetime); -} - -static inline void psa_extend_key_usage_flags(psa_key_usage_t *usage_flags) -{ - if (*usage_flags & PSA_KEY_USAGE_SIGN_HASH) { - *usage_flags |= PSA_KEY_USAGE_SIGN_MESSAGE; - } - - if (*usage_flags & PSA_KEY_USAGE_VERIFY_HASH) { - *usage_flags |= PSA_KEY_USAGE_VERIFY_MESSAGE; - } -} - -static inline void psa_set_key_usage_flags(psa_key_attributes_t *attributes, - psa_key_usage_t usage_flags) -{ - psa_extend_key_usage_flags(&usage_flags); - attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(usage) = usage_flags; -} - -static inline psa_key_usage_t psa_get_key_usage_flags( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(usage); -} - -static inline void psa_set_key_algorithm(psa_key_attributes_t *attributes, - psa_algorithm_t alg) -{ - attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(alg) = alg; -} - -static inline psa_algorithm_t psa_get_key_algorithm( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(policy).MBEDTLS_PRIVATE(alg); -} - -static inline void psa_set_key_type(psa_key_attributes_t *attributes, - psa_key_type_t type) -{ - attributes->MBEDTLS_PRIVATE(type) = type; -} - -static inline psa_key_type_t psa_get_key_type( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(type); -} - -static inline void psa_set_key_bits(psa_key_attributes_t *attributes, - size_t bits) -{ - if (bits > PSA_MAX_KEY_BITS) { - attributes->MBEDTLS_PRIVATE(bits) = PSA_KEY_BITS_TOO_LARGE; - } else { - attributes->MBEDTLS_PRIVATE(bits) = (psa_key_bits_t) bits; - } -} - -static inline size_t psa_get_key_bits( - const psa_key_attributes_t *attributes) -{ - return attributes->MBEDTLS_PRIVATE(bits); -} - -/** - * \brief The context for PSA interruptible hash signing. - */ -struct psa_sign_hash_interruptible_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_crypto_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - - psa_driver_sign_hash_interruptible_context_t MBEDTLS_PRIVATE(ctx); - - unsigned int MBEDTLS_PRIVATE(error_occurred) : 1; - - uint32_t MBEDTLS_PRIVATE(num_ops); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { 0 } -#else -#define PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT { 0, { 0 }, 0, 0 } -#endif - -static inline struct psa_sign_hash_interruptible_operation_s -psa_sign_hash_interruptible_operation_init(void) -{ - const struct psa_sign_hash_interruptible_operation_s v = - PSA_SIGN_HASH_INTERRUPTIBLE_OPERATION_INIT; - - return v; -} - -/** - * \brief The context for PSA interruptible hash verification. - */ -struct psa_verify_hash_interruptible_operation_s { -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) - mbedtls_psa_client_handle_t handle; -#else - /** Unique ID indicating which driver got assigned to do the - * operation. Since driver contexts are driver-specific, swapping - * drivers halfway through the operation is not supported. - * ID values are auto-generated in psa_crypto_driver_wrappers.h - * ID value zero means the context is not valid or not assigned to - * any driver (i.e. none of the driver contexts are active). */ - unsigned int MBEDTLS_PRIVATE(id); - - psa_driver_verify_hash_interruptible_context_t MBEDTLS_PRIVATE(ctx); - - unsigned int MBEDTLS_PRIVATE(error_occurred) : 1; - - uint32_t MBEDTLS_PRIVATE(num_ops); -#endif -}; - -#if defined(MBEDTLS_PSA_CRYPTO_CLIENT) && !defined(MBEDTLS_PSA_CRYPTO_C) -#define PSA_VERIFY_HASH_INTERRUPTIBLE_OPERATION_INIT { 0 } -#else -#define PSA_VERIFY_HASH_INTERRUPTIBLE_OPERATION_INIT { 0, { 0 }, 0, 0 } -#endif - -static inline struct psa_verify_hash_interruptible_operation_s -psa_verify_hash_interruptible_operation_init(void) -{ - const struct psa_verify_hash_interruptible_operation_s v = - PSA_VERIFY_HASH_INTERRUPTIBLE_OPERATION_INIT; - - return v; -} - -#ifdef __cplusplus -} -#endif - -#endif /* PSA_CRYPTO_STRUCT_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_types.h b/android/mbedtls/arm64-v8a/include/psa/crypto_types.h deleted file mode 100644 index f831486f4..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_types.h +++ /dev/null @@ -1,508 +0,0 @@ -/** - * \file psa/crypto_types.h - * - * \brief PSA cryptography module: type aliases. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. Drivers must include the appropriate driver - * header file. - * - * This file contains portable definitions of integral types for properties - * of cryptographic keys, designations of cryptographic algorithms, and - * error codes returned by the library. - * - * This header file does not declare any function. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_TYPES_H -#define PSA_CRYPTO_TYPES_H - -/* - * Include the build-time configuration information header. Here, we do not - * include `"mbedtls/build_info.h"` directly but `"psa/build_info.h"`, which - * is basically just an alias to it. This is to ease the maintenance of the - * TF-PSA-Crypto repository which has a different build system and - * configuration. - */ -#include "psa/build_info.h" - -/* Define the MBEDTLS_PRIVATE macro. */ -#include "mbedtls/private_access.h" - -#if defined(MBEDTLS_PSA_CRYPTO_PLATFORM_FILE) -#include MBEDTLS_PSA_CRYPTO_PLATFORM_FILE -#else -#include "crypto_platform.h" -#endif - -#include - -/** \defgroup error Error codes - * @{ - */ - -/** - * \brief Function return status. - * - * This is either #PSA_SUCCESS (which is zero), indicating success, - * or a small negative value indicating that an error occurred. Errors are - * encoded as one of the \c PSA_ERROR_xxx values defined here. */ -/* If #PSA_SUCCESS is already defined, it means that #psa_status_t - * is also defined in an external header, so prevent its multiple - * definition. - */ -#ifndef PSA_SUCCESS -typedef int32_t psa_status_t; -#endif - -/**@}*/ - -/** \defgroup crypto_types Key and algorithm types - * @{ - */ - -/** \brief Encoding of a key type. - * - * Values of this type are generally constructed by macros called - * `PSA_KEY_TYPE_xxx`. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint16_t psa_key_type_t; - -/** The type of PSA elliptic curve family identifiers. - * - * Values of this type are generally constructed by macros called - * `PSA_ECC_FAMILY_xxx`. - * - * The curve identifier is required to create an ECC key using the - * PSA_KEY_TYPE_ECC_KEY_PAIR() or PSA_KEY_TYPE_ECC_PUBLIC_KEY() - * macros. - * - * Values defined by this standard will never be in the range 0x80-0xff. - * Vendors who define additional families must use an encoding in this range. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint8_t psa_ecc_family_t; - -/** The type of PSA Diffie-Hellman group family identifiers. - * - * Values of this type are generally constructed by macros called - * `PSA_DH_FAMILY_xxx`. - * - * The group identifier is required to create a Diffie-Hellman key using the - * PSA_KEY_TYPE_DH_KEY_PAIR() or PSA_KEY_TYPE_DH_PUBLIC_KEY() - * macros. - * - * Values defined by this standard will never be in the range 0x80-0xff. - * Vendors who define additional families must use an encoding in this range. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint8_t psa_dh_family_t; - -/** \brief Encoding of a cryptographic algorithm. - * - * Values of this type are generally constructed by macros called - * `PSA_ALG_xxx`. - * - * For algorithms that can be applied to multiple key types, this type - * does not encode the key type. For example, for symmetric ciphers - * based on a block cipher, #psa_algorithm_t encodes the block cipher - * mode and the padding mode while the block cipher itself is encoded - * via #psa_key_type_t. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint32_t psa_algorithm_t; - -/**@}*/ - -/** \defgroup key_lifetimes Key lifetimes - * @{ - */ - -/** Encoding of key lifetimes. - * - * The lifetime of a key indicates where it is stored and what system actions - * may create and destroy it. - * - * Lifetime values have the following structure: - * - Bits 0-7 (#PSA_KEY_LIFETIME_GET_PERSISTENCE(\c lifetime)): - * persistence level. This value indicates what device management - * actions can cause it to be destroyed. In particular, it indicates - * whether the key is _volatile_ or _persistent_. - * See ::psa_key_persistence_t for more information. - * - Bits 8-31 (#PSA_KEY_LIFETIME_GET_LOCATION(\c lifetime)): - * location indicator. This value indicates which part of the system - * has access to the key material and can perform operations using the key. - * See ::psa_key_location_t for more information. - * - * Volatile keys are automatically destroyed when the application instance - * terminates or on a power reset of the device. Persistent keys are - * preserved until the application explicitly destroys them or until an - * integration-specific device management event occurs (for example, - * a factory reset). - * - * Persistent keys have a key identifier of type #mbedtls_svc_key_id_t. - * This identifier remains valid throughout the lifetime of the key, - * even if the application instance that created the key terminates. - * The application can call psa_open_key() to open a persistent key that - * it created previously. - * - * The default lifetime of a key is #PSA_KEY_LIFETIME_VOLATILE. The lifetime - * #PSA_KEY_LIFETIME_PERSISTENT is supported if persistent storage is - * available. Other lifetime values may be supported depending on the - * library configuration. - * - * Values of this type are generally constructed by macros called - * `PSA_KEY_LIFETIME_xxx`. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint32_t psa_key_lifetime_t; - -/** Encoding of key persistence levels. - * - * What distinguishes different persistence levels is what device management - * events may cause keys to be destroyed. _Volatile_ keys are destroyed - * by a power reset. Persistent keys may be destroyed by events such as - * a transfer of ownership or a factory reset. What management events - * actually affect persistent keys at different levels is outside the - * scope of the PSA Cryptography specification. - * - * The PSA Cryptography specification defines the following values of - * persistence levels: - * - \c 0 = #PSA_KEY_PERSISTENCE_VOLATILE: volatile key. - * A volatile key is automatically destroyed by the implementation when - * the application instance terminates. In particular, a volatile key - * is automatically destroyed on a power reset of the device. - * - \c 1 = #PSA_KEY_PERSISTENCE_DEFAULT: - * persistent key with a default lifetime. - * - \c 2-254: currently not supported by Mbed TLS. - * - \c 255 = #PSA_KEY_PERSISTENCE_READ_ONLY: - * read-only or write-once key. - * A key with this persistence level cannot be destroyed. - * Mbed TLS does not currently offer a way to create such keys, but - * integrations of Mbed TLS can use it for built-in keys that the - * application cannot modify (for example, a hardware unique key (HUK)). - * - * \note Key persistence levels are 8-bit values. Key management - * interfaces operate on lifetimes (type ::psa_key_lifetime_t) which - * encode the persistence as the lower 8 bits of a 32-bit value. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint8_t psa_key_persistence_t; - -/** Encoding of key location indicators. - * - * If an integration of Mbed TLS can make calls to external - * cryptoprocessors such as secure elements, the location of a key - * indicates which secure element performs the operations on the key. - * Depending on the design of the secure element, the key - * material may be stored either in the secure element, or - * in wrapped (encrypted) form alongside the key metadata in the - * primary local storage. - * - * The PSA Cryptography API specification defines the following values of - * location indicators: - * - \c 0: primary local storage. - * This location is always available. - * The primary local storage is typically the same storage area that - * contains the key metadata. - * - \c 1: primary secure element. - * Integrations of Mbed TLS should support this value if there is a secure - * element attached to the operating environment. - * As a guideline, secure elements may provide higher resistance against - * side channel and physical attacks than the primary local storage, but may - * have restrictions on supported key types, sizes, policies and operations - * and may have different performance characteristics. - * - \c 2-0x7fffff: other locations defined by a PSA specification. - * The PSA Cryptography API does not currently assign any meaning to these - * locations, but future versions of that specification or other PSA - * specifications may do so. - * - \c 0x800000-0xffffff: vendor-defined locations. - * No PSA specification will assign a meaning to locations in this range. - * - * \note Key location indicators are 24-bit values. Key management - * interfaces operate on lifetimes (type ::psa_key_lifetime_t) which - * encode the location as the upper 24 bits of a 32-bit value. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint32_t psa_key_location_t; - -/** Encoding of identifiers of persistent keys. - * - * - Applications may freely choose key identifiers in the range - * #PSA_KEY_ID_USER_MIN to #PSA_KEY_ID_USER_MAX. - * - The implementation may define additional key identifiers in the range - * #PSA_KEY_ID_VENDOR_MIN to #PSA_KEY_ID_VENDOR_MAX. - * - 0 is reserved as an invalid key identifier. - * - Key identifiers outside these ranges are reserved for future use. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to how values are allocated must require careful - * consideration to allow backward compatibility. - */ -typedef uint32_t psa_key_id_t; - -/** Encoding of key identifiers as seen inside the PSA Crypto implementation. - * - * When PSA Crypto is built as a library inside an application, this type - * is identical to #psa_key_id_t. When PSA Crypto is built as a service - * that can store keys on behalf of multiple clients, this type - * encodes the #psa_key_id_t value seen by each client application as - * well as extra information that identifies the client that owns - * the key. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -#if !defined(MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER) -typedef psa_key_id_t mbedtls_svc_key_id_t; - -#else /* MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER */ -/* Implementation-specific: The Mbed TLS library can be built as - * part of a multi-client service that exposes the PSA Cryptography API in each - * client and encodes the client identity in the key identifier argument of - * functions such as psa_open_key(). - */ -typedef struct { - psa_key_id_t MBEDTLS_PRIVATE(key_id); - mbedtls_key_owner_id_t MBEDTLS_PRIVATE(owner); -} mbedtls_svc_key_id_t; - -#endif /* !MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER */ - -/**@}*/ - -/** \defgroup policy Key policies - * @{ - */ - -/** \brief Encoding of permitted usage on a key. - * - * Values of this type are generally constructed as bitwise-ors of macros - * called `PSA_KEY_USAGE_xxx`. - * - * \note Values of this type are encoded in the persistent key store. - * Any changes to existing values will require bumping the storage - * format version and providing a translation when reading the old - * format. - */ -typedef uint32_t psa_key_usage_t; - -/**@}*/ - -/** \defgroup attributes Key attributes - * @{ - */ - -/** The type of a structure containing key attributes. - * - * This is an opaque structure that can represent the metadata of a key - * object. Metadata that can be stored in attributes includes: - * - The location of the key in storage, indicated by its key identifier - * and its lifetime. - * - The key's policy, comprising usage flags and a specification of - * the permitted algorithm(s). - * - Information about the key itself: the key type and its size. - * - Additional implementation-defined attributes. - * - * The actual key material is not considered an attribute of a key. - * Key attributes do not contain information that is generally considered - * highly confidential. - * - * An attribute structure works like a simple data structure where each function - * `psa_set_key_xxx` sets a field and the corresponding function - * `psa_get_key_xxx` retrieves the value of the corresponding field. - * However, a future version of the library may report values that are - * equivalent to the original one, but have a different encoding. Invalid - * values may be mapped to different, also invalid values. - * - * An attribute structure may contain references to auxiliary resources, - * for example pointers to allocated memory or indirect references to - * pre-calculated values. In order to free such resources, the application - * must call psa_reset_key_attributes(). As an exception, calling - * psa_reset_key_attributes() on an attribute structure is optional if - * the structure has only been modified by the following functions - * since it was initialized or last reset with psa_reset_key_attributes(): - * - psa_set_key_id() - * - psa_set_key_lifetime() - * - psa_set_key_type() - * - psa_set_key_bits() - * - psa_set_key_usage_flags() - * - psa_set_key_algorithm() - * - * Before calling any function on a key attribute structure, the application - * must initialize it by any of the following means: - * - Set the structure to all-bits-zero, for example: - * \code - * psa_key_attributes_t attributes; - * memset(&attributes, 0, sizeof(attributes)); - * \endcode - * - Initialize the structure to logical zero values, for example: - * \code - * psa_key_attributes_t attributes = {0}; - * \endcode - * - Initialize the structure to the initializer #PSA_KEY_ATTRIBUTES_INIT, - * for example: - * \code - * psa_key_attributes_t attributes = PSA_KEY_ATTRIBUTES_INIT; - * \endcode - * - Assign the result of the function psa_key_attributes_init() - * to the structure, for example: - * \code - * psa_key_attributes_t attributes; - * attributes = psa_key_attributes_init(); - * \endcode - * - * A freshly initialized attribute structure contains the following - * values: - * - * - lifetime: #PSA_KEY_LIFETIME_VOLATILE. - * - key identifier: 0 (which is not a valid key identifier). - * - type: \c 0 (meaning that the type is unspecified). - * - key size: \c 0 (meaning that the size is unspecified). - * - usage flags: \c 0 (which allows no usage except exporting a public key). - * - algorithm: \c 0 (which allows no cryptographic usage, but allows - * exporting). - * - * A typical sequence to create a key is as follows: - * -# Create and initialize an attribute structure. - * -# If the key is persistent, call psa_set_key_id(). - * Also call psa_set_key_lifetime() to place the key in a non-default - * location. - * -# Set the key policy with psa_set_key_usage_flags() and - * psa_set_key_algorithm(). - * -# Set the key type with psa_set_key_type(). - * Skip this step if copying an existing key with psa_copy_key(). - * -# When generating a random key with psa_generate_key() or deriving a key - * with psa_key_derivation_output_key(), set the desired key size with - * psa_set_key_bits(). - * -# Call a key creation function: psa_import_key(), psa_generate_key(), - * psa_key_derivation_output_key() or psa_copy_key(). This function reads - * the attribute structure, creates a key with these attributes, and - * outputs a key identifier to the newly created key. - * -# The attribute structure is now no longer necessary. - * You may call psa_reset_key_attributes(), although this is optional - * with the workflow presented here because the attributes currently - * defined in this specification do not require any additional resources - * beyond the structure itself. - * - * A typical sequence to query a key's attributes is as follows: - * -# Call psa_get_key_attributes(). - * -# Call `psa_get_key_xxx` functions to retrieve the attribute(s) that - * you are interested in. - * -# Call psa_reset_key_attributes() to free any resources that may be - * used by the attribute structure. - * - * Once a key has been created, it is impossible to change its attributes. - */ -typedef struct psa_key_attributes_s psa_key_attributes_t; - - -#ifndef __DOXYGEN_ONLY__ -#if defined(MBEDTLS_PSA_CRYPTO_SE_C) -/* Mbed TLS defines this type in crypto_types.h because it is also - * visible to applications through an implementation-specific extension. - * For the PSA Cryptography specification, this type is only visible - * via crypto_se_driver.h. */ -typedef uint64_t psa_key_slot_number_t; -#endif /* MBEDTLS_PSA_CRYPTO_SE_C */ -#endif /* !__DOXYGEN_ONLY__ */ - -/**@}*/ - -/** \defgroup derivation Key derivation - * @{ - */ - -/** \brief Encoding of the step of a key derivation. - * - * Values of this type are generally constructed by macros called - * `PSA_KEY_DERIVATION_INPUT_xxx`. - */ -typedef uint16_t psa_key_derivation_step_t; - -/** \brief Custom parameters for key generation or key derivation. - * - * This is a structure type with at least the following field: - * - * - \c flags: an unsigned integer type. 0 for the default production parameters. - * - * Functions that take such a structure as input also take an associated - * input buffer \c custom_data of length \c custom_data_length. - * - * The interpretation of this structure and the associated \c custom_data - * parameter depend on the type of the created key. - * - * - #PSA_KEY_TYPE_RSA_KEY_PAIR: - * - \c flags: must be 0. - * - \c custom_data: the public exponent, in little-endian order. - * This must be an odd integer and must not be 1. - * Implementations must support 65537, should support 3 and may - * support other values. - * When not using a driver, Mbed TLS supports values up to \c INT_MAX. - * If this is empty, the default value 65537 is used. - * - Other key types: reserved for future use. \c flags must be 0. - */ -typedef struct psa_custom_key_parameters_s psa_custom_key_parameters_t; - -/** \brief Custom parameters for key generation or key derivation. - * - * This is a structure type with at least the following fields: - * - * - \c flags: an unsigned integer type. 0 for the default production parameters. - * - \c data: a flexible array of bytes. - * - * The interpretation of this structure depend on the type of the - * created key. - * - * - #PSA_KEY_TYPE_RSA_KEY_PAIR: - * - \c flags: must be 0. - * - \c data: the public exponent, in little-endian order. - * This must be an odd integer and must not be 1. - * Implementations must support 65537, should support 3 and may - * support other values. - * When not using a driver, Mbed TLS supports values up to \c INT_MAX. - * If this is empty or if the custom production parameters are omitted - * altogether, the default value 65537 is used. - * - Other key types: reserved for future use. \c flags must be 0. - * - */ -typedef struct psa_key_production_parameters_s psa_key_production_parameters_t; - -/**@}*/ - -#endif /* PSA_CRYPTO_TYPES_H */ diff --git a/android/mbedtls/arm64-v8a/include/psa/crypto_values.h b/android/mbedtls/arm64-v8a/include/psa/crypto_values.h deleted file mode 100644 index 1d678dbfc..000000000 --- a/android/mbedtls/arm64-v8a/include/psa/crypto_values.h +++ /dev/null @@ -1,2783 +0,0 @@ -/** - * \file psa/crypto_values.h - * - * \brief PSA cryptography module: macros to build and analyze integer values. - * - * \note This file may not be included directly. Applications must - * include psa/crypto.h. Drivers must include the appropriate driver - * header file. - * - * This file contains portable definitions of macros to build and analyze - * values of integral types that encode properties of cryptographic keys, - * designations of cryptographic algorithms, and error codes returned by - * the library. - * - * Note that many of the constants defined in this file are embedded in - * the persistent key store, as part of key metadata (including usage - * policies). As a consequence, they must not be changed (unless the storage - * format version changes). - * - * This header file only defines preprocessor macros. - */ -/* - * Copyright The Mbed TLS Contributors - * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later - */ - -#ifndef PSA_CRYPTO_VALUES_H -#define PSA_CRYPTO_VALUES_H -#include "mbedtls/private_access.h" - -/** \defgroup error Error codes - * @{ - */ - -/* PSA error codes */ - -/* Error codes are standardized across PSA domains (framework, crypto, storage, - * etc.). Do not change the values in this section or even the expansions - * of each macro: it must be possible to `#include` both this header - * and some other PSA component's headers in the same C source, - * which will lead to duplicate definitions of the `PSA_SUCCESS` and - * `PSA_ERROR_xxx` macros, which is ok if and only if the macros expand - * to the same sequence of tokens. - * - * If you must add a new - * value, check with the Arm PSA framework group to pick one that other - * domains aren't already using. */ - -/* Tell uncrustify not to touch the constant definitions, otherwise - * it might change the spacing to something that is not PSA-compliant - * (e.g. adding a space after casts). - * - * *INDENT-OFF* - */ - -/** The action was completed successfully. */ -#define PSA_SUCCESS ((psa_status_t)0) - -/** An error occurred that does not correspond to any defined - * failure cause. - * - * Implementations may use this error code if none of the other standard - * error codes are applicable. */ -#define PSA_ERROR_GENERIC_ERROR ((psa_status_t)-132) - -/** The requested operation or a parameter is not supported - * by this implementation. - * - * Implementations should return this error code when an enumeration - * parameter such as a key type, algorithm, etc. is not recognized. - * If a combination of parameters is recognized and identified as - * not valid, return #PSA_ERROR_INVALID_ARGUMENT instead. */ -#define PSA_ERROR_NOT_SUPPORTED ((psa_status_t)-134) - -/** The requested action is denied by a policy. - * - * Implementations should return this error code when the parameters - * are recognized as valid and supported, and a policy explicitly - * denies the requested operation. - * - * If a subset of the parameters of a function call identify a - * forbidden operation, and another subset of the parameters are - * not valid or not supported, it is unspecified whether the function - * returns #PSA_ERROR_NOT_PERMITTED, #PSA_ERROR_NOT_SUPPORTED or - * #PSA_ERROR_INVALID_ARGUMENT. */ -#define PSA_ERROR_NOT_PERMITTED ((psa_status_t)-133) - -/** An output buffer is too small. - * - * Applications can call the \c PSA_xxx_SIZE macro listed in the function - * description to determine a sufficient buffer size. - * - * Implementations should preferably return this error code only - * in cases when performing the operation with a larger output - * buffer would succeed. However implementations may return this - * error if a function has invalid or unsupported parameters in addition - * to the parameters that determine the necessary output buffer size. */ -#define PSA_ERROR_BUFFER_TOO_SMALL ((psa_status_t)-138) - -/** Asking for an item that already exists - * - * Implementations should return this error, when attempting - * to write an item (like a key) that already exists. */ -#define PSA_ERROR_ALREADY_EXISTS ((psa_status_t)-139) - -/** Asking for an item that doesn't exist - * - * Implementations should return this error, if a requested item (like - * a key) does not exist. */ -#define PSA_ERROR_DOES_NOT_EXIST ((psa_status_t)-140) - -/** The requested action cannot be performed in the current state. - * - * Multipart operations return this error when one of the - * functions is called out of sequence. Refer to the function - * descriptions for permitted sequencing of functions. - * - * Implementations shall not return this error code to indicate - * that a key either exists or not, - * but shall instead return #PSA_ERROR_ALREADY_EXISTS or #PSA_ERROR_DOES_NOT_EXIST - * as applicable. - * - * Implementations shall not return this error code to indicate that a - * key identifier is invalid, but shall return #PSA_ERROR_INVALID_HANDLE - * instead. */ -#define PSA_ERROR_BAD_STATE ((psa_status_t)-137) - -/** The parameters passed to the function are invalid. - * - * Implementations may return this error any time a parameter or - * combination of parameters are recognized as invalid. - * - * Implementations shall not return this error code to indicate that a - * key identifier is invalid, but shall return #PSA_ERROR_INVALID_HANDLE - * instead. - */ -#define PSA_ERROR_INVALID_ARGUMENT ((psa_status_t)-135) - -/** There is not enough runtime memory. - * - * If the action is carried out across multiple security realms, this - * error can refer to available memory in any of the security realms. */ -#define PSA_ERROR_INSUFFICIENT_MEMORY ((psa_status_t)-141) - -/** There is not enough persistent storage. - * - * Functions that modify the key storage return this error code if - * there is insufficient storage space on the host media. In addition, - * many functions that do not otherwise access storage may return this - * error code if the implementation requires a mandatory log entry for - * the requested action and the log storage space is full. */ -#define PSA_ERROR_INSUFFICIENT_STORAGE ((psa_status_t)-142) - -/** There was a communication failure inside the implementation. - * - * This can indicate a communication failure between the application - * and an external cryptoprocessor or between the cryptoprocessor and - * an external volatile or persistent memory. A communication failure - * may be transient or permanent depending on the cause. - * - * \warning If a function returns this error, it is undetermined - * whether the requested action has completed or not. Implementations - * should return #PSA_SUCCESS on successful completion whenever - * possible, however functions may return #PSA_ERROR_COMMUNICATION_FAILURE - * if the requested action was completed successfully in an external - * cryptoprocessor but there was a breakdown of communication before - * the cryptoprocessor could report the status to the application. - */ -#define PSA_ERROR_COMMUNICATION_FAILURE ((psa_status_t)-145) - -/** There was a storage failure that may have led to data loss. - * - * This error indicates that some persistent storage is corrupted. - * It should not be used for a corruption of volatile memory - * (use #PSA_ERROR_CORRUPTION_DETECTED), for a communication error - * between the cryptoprocessor and its external storage (use - * #PSA_ERROR_COMMUNICATION_FAILURE), or when the storage is - * in a valid state but is full (use #PSA_ERROR_INSUFFICIENT_STORAGE). - * - * Note that a storage failure does not indicate that any data that was - * previously read is invalid. However this previously read data may no - * longer be readable from storage. - * - * When a storage failure occurs, it is no longer possible to ensure - * the global integrity of the keystore. Depending on the global - * integrity guarantees offered by the implementation, access to other - * data may or may not fail even if the data is still readable but - * its integrity cannot be guaranteed. - * - * Implementations should only use this error code to report a - * permanent storage corruption. However application writers should - * keep in mind that transient errors while reading the storage may be - * reported using this error code. */ -#define PSA_ERROR_STORAGE_FAILURE ((psa_status_t)-146) - -/** A hardware failure was detected. - * - * A hardware failure may be transient or permanent depending on the - * cause. */ -#define PSA_ERROR_HARDWARE_FAILURE ((psa_status_t)-147) - -/** A tampering attempt was detected. - * - * If an application receives this error code, there is no guarantee - * that previously accessed or computed data was correct and remains - * confidential. Applications should not perform any security function - * and should enter a safe failure state. - * - * Implementations may return this error code if they detect an invalid - * state that cannot happen during normal operation and that indicates - * that the implementation's security guarantees no longer hold. Depending - * on the implementation architecture and on its security and safety goals, - * the implementation may forcibly terminate the application. - * - * This error code is intended as a last resort when a security breach - * is detected and it is unsure whether the keystore data is still - * protected. Implementations shall only return this error code - * to report an alarm from a tampering detector, to indicate that - * the confidentiality of stored data can no longer be guaranteed, - * or to indicate that the integrity of previously returned data is now - * considered compromised. Implementations shall not use this error code - * to indicate a hardware failure that merely makes it impossible to - * perform the requested operation (use #PSA_ERROR_COMMUNICATION_FAILURE, - * #PSA_ERROR_STORAGE_FAILURE, #PSA_ERROR_HARDWARE_FAILURE, - * #PSA_ERROR_INSUFFICIENT_ENTROPY or other applicable error code - * instead). - * - * This error indicates an attack against the application. Implementations - * shall not return this error code as a consequence of the behavior of - * the application itself. */ -#define PSA_ERROR_CORRUPTION_DETECTED ((psa_status_t)-151) - -/** There is not enough entropy to generate random data needed - * for the requested action. - * - * This error indicates a failure of a hardware random generator. - * Application writers should note that this error can be returned not - * only by functions whose purpose is to generate random data, such - * as key, IV or nonce generation, but also by functions that execute - * an algorithm with a randomized result, as well as functions that - * use randomization of intermediate computations as a countermeasure - * to certain attacks. - * - * Implementations should avoid returning this error after psa_crypto_init() - * has succeeded. Implementations should generate sufficient - * entropy during initialization and subsequently use a cryptographically - * secure pseudorandom generator (PRNG). However implementations may return - * this error at any time if a policy requires the PRNG to be reseeded - * during normal operation. */ -#define PSA_ERROR_INSUFFICIENT_ENTROPY ((psa_status_t)-148) - -/** The signature, MAC or hash is incorrect. - * - * Verification functions return this error if the verification - * calculations completed successfully, and the value to be verified - * was determined to be incorrect. - * - * If the value to verify has an invalid size, implementations may return - * either #PSA_ERROR_INVALID_ARGUMENT or #PSA_ERROR_INVALID_SIGNATURE. */ -#define PSA_ERROR_INVALID_SIGNATURE ((psa_status_t)-149) - -/** The decrypted padding is incorrect. - * - * \warning In some protocols, when decrypting data, it is essential that - * the behavior of the application does not depend on whether the padding - * is correct, down to precise timing. Applications should prefer - * protocols that use authenticated encryption rather than plain - * encryption. If the application must perform a decryption of - * unauthenticated data, the application writer should take care not - * to reveal whether the padding is invalid. - * - * Implementations should strive to make valid and invalid padding - * as close as possible to indistinguishable to an external observer. - * In particular, the timing of a decryption operation should not - * depend on the validity of the padding. */ -#define PSA_ERROR_INVALID_PADDING ((psa_status_t)-150) - -/** Return this error when there's insufficient data when attempting - * to read from a resource. */ -#define PSA_ERROR_INSUFFICIENT_DATA ((psa_status_t)-143) - -/** This can be returned if a function can no longer operate correctly. - * For example, if an essential initialization operation failed or - * a mutex operation failed. */ -#define PSA_ERROR_SERVICE_FAILURE ((psa_status_t)-144) - -/** The key identifier is not valid. See also :ref:\`key-handles\`. - */ -#define PSA_ERROR_INVALID_HANDLE ((psa_status_t)-136) - -/** Stored data has been corrupted. - * - * This error indicates that some persistent storage has suffered corruption. - * It does not indicate the following situations, which have specific error - * codes: - * - * - A corruption of volatile memory - use #PSA_ERROR_CORRUPTION_DETECTED. - * - A communication error between the cryptoprocessor and its external - * storage - use #PSA_ERROR_COMMUNICATION_FAILURE. - * - When the storage is in a valid state but is full - use - * #PSA_ERROR_INSUFFICIENT_STORAGE. - * - When the storage fails for other reasons - use - * #PSA_ERROR_STORAGE_FAILURE. - * - When the stored data is not valid - use #PSA_ERROR_DATA_INVALID. - * - * \note A storage corruption does not indicate that any data that was - * previously read is invalid. However this previously read data might no - * longer be readable from storage. - * - * When a storage failure occurs, it is no longer possible to ensure the - * global integrity of the keystore. - */ -#define PSA_ERROR_DATA_CORRUPT ((psa_status_t)-152) - -/** Data read from storage is not valid for the implementation. - * - * This error indicates that some data read from storage does not have a valid - * format. It does not indicate the following situations, which have specific - * error codes: - * - * - When the storage or stored data is corrupted - use #PSA_ERROR_DATA_CORRUPT - * - When the storage fails for other reasons - use #PSA_ERROR_STORAGE_FAILURE - * - An invalid argument to the API - use #PSA_ERROR_INVALID_ARGUMENT - * - * This error is typically a result of either storage corruption on a - * cleartext storage backend, or an attempt to read data that was - * written by an incompatible version of the library. - */ -#define PSA_ERROR_DATA_INVALID ((psa_status_t)-153) - -/** The function that returns this status is defined as interruptible and - * still has work to do, thus the user should call the function again with the - * same operation context until it either returns #PSA_SUCCESS or any other - * error. This is not an error per se, more a notification of status. - */ -#define PSA_OPERATION_INCOMPLETE ((psa_status_t)-248) - -/* *INDENT-ON* */ - -/**@}*/ - -/** \defgroup crypto_types Key and algorithm types - * @{ - */ - -/* Note that key type values, including ECC family and DH group values, are - * embedded in the persistent key store, as part of key metadata. As a - * consequence, they must not be changed (unless the storage format version - * changes). - */ - -/** An invalid key type value. - * - * Zero is not the encoding of any key type. - */ -#define PSA_KEY_TYPE_NONE ((psa_key_type_t) 0x0000) - -/** Vendor-defined key type flag. - * - * Key types defined by this standard will never have the - * #PSA_KEY_TYPE_VENDOR_FLAG bit set. Vendors who define additional key types - * must use an encoding with the #PSA_KEY_TYPE_VENDOR_FLAG bit set and should - * respect the bitwise structure used by standard encodings whenever practical. - */ -#define PSA_KEY_TYPE_VENDOR_FLAG ((psa_key_type_t) 0x8000) - -#define PSA_KEY_TYPE_CATEGORY_MASK ((psa_key_type_t) 0x7000) -#define PSA_KEY_TYPE_CATEGORY_RAW ((psa_key_type_t) 0x1000) -#define PSA_KEY_TYPE_CATEGORY_SYMMETRIC ((psa_key_type_t) 0x2000) -#define PSA_KEY_TYPE_CATEGORY_PUBLIC_KEY ((psa_key_type_t) 0x4000) -#define PSA_KEY_TYPE_CATEGORY_KEY_PAIR ((psa_key_type_t) 0x7000) - -#define PSA_KEY_TYPE_CATEGORY_FLAG_PAIR ((psa_key_type_t) 0x3000) - -/** Whether a key type is vendor-defined. - * - * See also #PSA_KEY_TYPE_VENDOR_FLAG. - */ -#define PSA_KEY_TYPE_IS_VENDOR_DEFINED(type) \ - (((type) & PSA_KEY_TYPE_VENDOR_FLAG) != 0) - -/** Whether a key type is an unstructured array of bytes. - * - * This encompasses both symmetric keys and non-key data. - */ -#define PSA_KEY_TYPE_IS_UNSTRUCTURED(type) \ - (((type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_RAW || \ - ((type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_SYMMETRIC) - -/** Whether a key type is asymmetric: either a key pair or a public key. */ -#define PSA_KEY_TYPE_IS_ASYMMETRIC(type) \ - (((type) & PSA_KEY_TYPE_CATEGORY_MASK \ - & ~PSA_KEY_TYPE_CATEGORY_FLAG_PAIR) == \ - PSA_KEY_TYPE_CATEGORY_PUBLIC_KEY) -/** Whether a key type is the public part of a key pair. */ -#define PSA_KEY_TYPE_IS_PUBLIC_KEY(type) \ - (((type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_PUBLIC_KEY) -/** Whether a key type is a key pair containing a private part and a public - * part. */ -#define PSA_KEY_TYPE_IS_KEY_PAIR(type) \ - (((type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_KEY_PAIR) -/** The key pair type corresponding to a public key type. - * - * You may also pass a key pair type as \p type, it will be left unchanged. - * - * \param type A public key type or key pair type. - * - * \return The corresponding key pair type. - * If \p type is not a public key or a key pair, - * the return value is undefined. - */ -#define PSA_KEY_TYPE_KEY_PAIR_OF_PUBLIC_KEY(type) \ - ((type) | PSA_KEY_TYPE_CATEGORY_FLAG_PAIR) -/** The public key type corresponding to a key pair type. - * - * You may also pass a public key type as \p type, it will be left unchanged. - * - * \param type A public key type or key pair type. - * - * \return The corresponding public key type. - * If \p type is not a public key or a key pair, - * the return value is undefined. - */ -#define PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(type) \ - ((type) & ~PSA_KEY_TYPE_CATEGORY_FLAG_PAIR) - -/** Raw data. - * - * A "key" of this type cannot be used for any cryptographic operation. - * Applications may use this type to store arbitrary data in the keystore. */ -#define PSA_KEY_TYPE_RAW_DATA ((psa_key_type_t) 0x1001) - -/** HMAC key. - * - * The key policy determines which underlying hash algorithm the key can be - * used for. - * - * HMAC keys should generally have the same size as the underlying hash. - * This size can be calculated with #PSA_HASH_LENGTH(\c alg) where - * \c alg is the HMAC algorithm or the underlying hash algorithm. */ -#define PSA_KEY_TYPE_HMAC ((psa_key_type_t) 0x1100) - -/** A secret for key derivation. - * - * This key type is for high-entropy secrets only. For low-entropy secrets, - * #PSA_KEY_TYPE_PASSWORD should be used instead. - * - * These keys can be used as the #PSA_KEY_DERIVATION_INPUT_SECRET or - * #PSA_KEY_DERIVATION_INPUT_PASSWORD input of key derivation algorithms. - * - * The key policy determines which key derivation algorithm the key - * can be used for. - */ -#define PSA_KEY_TYPE_DERIVE ((psa_key_type_t) 0x1200) - -/** A low-entropy secret for password hashing or key derivation. - * - * This key type is suitable for passwords and passphrases which are typically - * intended to be memorizable by humans, and have a low entropy relative to - * their size. It can be used for randomly generated or derived keys with - * maximum or near-maximum entropy, but #PSA_KEY_TYPE_DERIVE is more suitable - * for such keys. It is not suitable for passwords with extremely low entropy, - * such as numerical PINs. - * - * These keys can be used as the #PSA_KEY_DERIVATION_INPUT_PASSWORD input of - * key derivation algorithms. Algorithms that accept such an input were - * designed to accept low-entropy secret and are known as password hashing or - * key stretching algorithms. - * - * These keys cannot be used as the #PSA_KEY_DERIVATION_INPUT_SECRET input of - * key derivation algorithms, as the algorithms that take such an input expect - * it to be high-entropy. - * - * The key policy determines which key derivation algorithm the key can be - * used for, among the permissible subset defined above. - */ -#define PSA_KEY_TYPE_PASSWORD ((psa_key_type_t) 0x1203) - -/** A secret value that can be used to verify a password hash. - * - * The key policy determines which key derivation algorithm the key - * can be used for, among the same permissible subset as for - * #PSA_KEY_TYPE_PASSWORD. - */ -#define PSA_KEY_TYPE_PASSWORD_HASH ((psa_key_type_t) 0x1205) - -/** A secret value that can be used in when computing a password hash. - * - * The key policy determines which key derivation algorithm the key - * can be used for, among the subset of algorithms that can use pepper. - */ -#define PSA_KEY_TYPE_PEPPER ((psa_key_type_t) 0x1206) - -/** Key for a cipher, AEAD or MAC algorithm based on the AES block cipher. - * - * The size of the key can be 16 bytes (AES-128), 24 bytes (AES-192) or - * 32 bytes (AES-256). - */ -#define PSA_KEY_TYPE_AES ((psa_key_type_t) 0x2400) - -/** Key for a cipher, AEAD or MAC algorithm based on the - * ARIA block cipher. */ -#define PSA_KEY_TYPE_ARIA ((psa_key_type_t) 0x2406) - -/** Key for a cipher or MAC algorithm based on DES or 3DES (Triple-DES). - * - * The size of the key can be 64 bits (single DES), 128 bits (2-key 3DES) or - * 192 bits (3-key 3DES). - * - * Note that single DES and 2-key 3DES are weak and strongly - * deprecated and should only be used to decrypt legacy data. 3-key 3DES - * is weak and deprecated and should only be used in legacy protocols. - */ -#define PSA_KEY_TYPE_DES ((psa_key_type_t) 0x2301) - -/** Key for a cipher, AEAD or MAC algorithm based on the - * Camellia block cipher. */ -#define PSA_KEY_TYPE_CAMELLIA ((psa_key_type_t) 0x2403) - -/** Key for the ChaCha20 stream cipher or the Chacha20-Poly1305 AEAD algorithm. - * - * ChaCha20 and the ChaCha20_Poly1305 construction are defined in RFC 7539. - * - * \note For ChaCha20 and ChaCha20_Poly1305, Mbed TLS only supports - * 12-byte nonces. - * - * \note For ChaCha20, the initial counter value is 0. To encrypt or decrypt - * with the initial counter value 1, you can process and discard a - * 64-byte block before the real data. - */ -#define PSA_KEY_TYPE_CHACHA20 ((psa_key_type_t) 0x2004) - -/** RSA public key. - * - * The size of an RSA key is the bit size of the modulus. - */ -#define PSA_KEY_TYPE_RSA_PUBLIC_KEY ((psa_key_type_t) 0x4001) -/** RSA key pair (private and public key). - * - * The size of an RSA key is the bit size of the modulus. - */ -#define PSA_KEY_TYPE_RSA_KEY_PAIR ((psa_key_type_t) 0x7001) -/** Whether a key type is an RSA key (pair or public-only). */ -#define PSA_KEY_TYPE_IS_RSA(type) \ - (PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(type) == PSA_KEY_TYPE_RSA_PUBLIC_KEY) - -#define PSA_KEY_TYPE_ECC_PUBLIC_KEY_BASE ((psa_key_type_t) 0x4100) -#define PSA_KEY_TYPE_ECC_KEY_PAIR_BASE ((psa_key_type_t) 0x7100) -#define PSA_KEY_TYPE_ECC_CURVE_MASK ((psa_key_type_t) 0x00ff) -/** Elliptic curve key pair. - * - * The size of an elliptic curve key is the bit size associated with the curve, - * i.e. the bit size of *q* for a curve over a field *Fq*. - * See the documentation of `PSA_ECC_FAMILY_xxx` curve families for details. - * - * \param curve A value of type ::psa_ecc_family_t that - * identifies the ECC curve to be used. - */ -#define PSA_KEY_TYPE_ECC_KEY_PAIR(curve) \ - (PSA_KEY_TYPE_ECC_KEY_PAIR_BASE | (curve)) -/** Elliptic curve public key. - * - * The size of an elliptic curve public key is the same as the corresponding - * private key (see #PSA_KEY_TYPE_ECC_KEY_PAIR and the documentation of - * `PSA_ECC_FAMILY_xxx` curve families). - * - * \param curve A value of type ::psa_ecc_family_t that - * identifies the ECC curve to be used. - */ -#define PSA_KEY_TYPE_ECC_PUBLIC_KEY(curve) \ - (PSA_KEY_TYPE_ECC_PUBLIC_KEY_BASE | (curve)) - -/** Whether a key type is an elliptic curve key (pair or public-only). */ -#define PSA_KEY_TYPE_IS_ECC(type) \ - ((PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(type) & \ - ~PSA_KEY_TYPE_ECC_CURVE_MASK) == PSA_KEY_TYPE_ECC_PUBLIC_KEY_BASE) -/** Whether a key type is an elliptic curve key pair. */ -#define PSA_KEY_TYPE_IS_ECC_KEY_PAIR(type) \ - (((type) & ~PSA_KEY_TYPE_ECC_CURVE_MASK) == \ - PSA_KEY_TYPE_ECC_KEY_PAIR_BASE) -/** Whether a key type is an elliptic curve public key. */ -#define PSA_KEY_TYPE_IS_ECC_PUBLIC_KEY(type) \ - (((type) & ~PSA_KEY_TYPE_ECC_CURVE_MASK) == \ - PSA_KEY_TYPE_ECC_PUBLIC_KEY_BASE) - -/** Extract the curve from an elliptic curve key type. */ -#define PSA_KEY_TYPE_ECC_GET_FAMILY(type) \ - ((psa_ecc_family_t) (PSA_KEY_TYPE_IS_ECC(type) ? \ - ((type) & PSA_KEY_TYPE_ECC_CURVE_MASK) : \ - 0)) - -/** Check if the curve of given family is Weierstrass elliptic curve. */ -#define PSA_ECC_FAMILY_IS_WEIERSTRASS(family) ((family & 0xc0) == 0) - -/** SEC Koblitz curves over prime fields. - * - * This family comprises the following curves: - * secp192k1, secp224k1, secp256k1. - * They are defined in _Standards for Efficient Cryptography_, - * _SEC 2: Recommended Elliptic Curve Domain Parameters_. - * https://www.secg.org/sec2-v2.pdf - * - * \note For secp224k1, the bit-size is 225 (size of a private value). - * - * \note Mbed TLS only supports secp192k1 and secp256k1. - */ -#define PSA_ECC_FAMILY_SECP_K1 ((psa_ecc_family_t) 0x17) - -/** SEC random curves over prime fields. - * - * This family comprises the following curves: - * secp192r1, secp224r1, secp256r1, secp384r1, secp521r1. - * They are defined in _Standards for Efficient Cryptography_, - * _SEC 2: Recommended Elliptic Curve Domain Parameters_. - * https://www.secg.org/sec2-v2.pdf - */ -#define PSA_ECC_FAMILY_SECP_R1 ((psa_ecc_family_t) 0x12) -/* SECP160R2 (SEC2 v1, obsolete, not supported in Mbed TLS) */ -#define PSA_ECC_FAMILY_SECP_R2 ((psa_ecc_family_t) 0x1b) - -/** SEC Koblitz curves over binary fields. - * - * This family comprises the following curves: - * sect163k1, sect233k1, sect239k1, sect283k1, sect409k1, sect571k1. - * They are defined in _Standards for Efficient Cryptography_, - * _SEC 2: Recommended Elliptic Curve Domain Parameters_. - * https://www.secg.org/sec2-v2.pdf - * - * \note Mbed TLS does not support any curve in this family. - */ -#define PSA_ECC_FAMILY_SECT_K1 ((psa_ecc_family_t) 0x27) - -/** SEC random curves over binary fields. - * - * This family comprises the following curves: - * sect163r1, sect233r1, sect283r1, sect409r1, sect571r1. - * They are defined in _Standards for Efficient Cryptography_, - * _SEC 2: Recommended Elliptic Curve Domain Parameters_. - * https://www.secg.org/sec2-v2.pdf - * - * \note Mbed TLS does not support any curve in this family. - */ -#define PSA_ECC_FAMILY_SECT_R1 ((psa_ecc_family_t) 0x22) - -/** SEC additional random curves over binary fields. - * - * This family comprises the following curve: - * sect163r2. - * It is defined in _Standards for Efficient Cryptography_, - * _SEC 2: Recommended Elliptic Curve Domain Parameters_. - * https://www.secg.org/sec2-v2.pdf - * - * \note Mbed TLS does not support any curve in this family. - */ -#define PSA_ECC_FAMILY_SECT_R2 ((psa_ecc_family_t) 0x2b) - -/** Brainpool P random curves. - * - * This family comprises the following curves: - * brainpoolP160r1, brainpoolP192r1, brainpoolP224r1, brainpoolP256r1, - * brainpoolP320r1, brainpoolP384r1, brainpoolP512r1. - * It is defined in RFC 5639. - * - * \note Mbed TLS only supports the 256-bit, 384-bit and 512-bit curves - * in this family. - */ -#define PSA_ECC_FAMILY_BRAINPOOL_P_R1 ((psa_ecc_family_t) 0x30) - -/** Curve25519 and Curve448. - * - * This family comprises the following Montgomery curves: - * - 255-bit: Bernstein et al., - * _Curve25519: new Diffie-Hellman speed records_, LNCS 3958, 2006. - * The algorithm #PSA_ALG_ECDH performs X25519 when used with this curve. - * - 448-bit: Hamburg, - * _Ed448-Goldilocks, a new elliptic curve_, NIST ECC Workshop, 2015. - * The algorithm #PSA_ALG_ECDH performs X448 when used with this curve. - */ -#define PSA_ECC_FAMILY_MONTGOMERY ((psa_ecc_family_t) 0x41) - -/** The twisted Edwards curves Ed25519 and Ed448. - * - * These curves are suitable for EdDSA (#PSA_ALG_PURE_EDDSA for both curves, - * #PSA_ALG_ED25519PH for the 255-bit curve, - * #PSA_ALG_ED448PH for the 448-bit curve). - * - * This family comprises the following twisted Edwards curves: - * - 255-bit: Edwards25519, the twisted Edwards curve birationally equivalent - * to Curve25519. - * Bernstein et al., _Twisted Edwards curves_, Africacrypt 2008. - * - 448-bit: Edwards448, the twisted Edwards curve birationally equivalent - * to Curve448. - * Hamburg, _Ed448-Goldilocks, a new elliptic curve_, NIST ECC Workshop, 2015. - * - * \note Mbed TLS does not support Edwards curves yet. - */ -#define PSA_ECC_FAMILY_TWISTED_EDWARDS ((psa_ecc_family_t) 0x42) - -#define PSA_KEY_TYPE_DH_PUBLIC_KEY_BASE ((psa_key_type_t) 0x4200) -#define PSA_KEY_TYPE_DH_KEY_PAIR_BASE ((psa_key_type_t) 0x7200) -#define PSA_KEY_TYPE_DH_GROUP_MASK ((psa_key_type_t) 0x00ff) -/** Diffie-Hellman key pair. - * - * \param group A value of type ::psa_dh_family_t that identifies the - * Diffie-Hellman group to be used. - */ -#define PSA_KEY_TYPE_DH_KEY_PAIR(group) \ - (PSA_KEY_TYPE_DH_KEY_PAIR_BASE | (group)) -/** Diffie-Hellman public key. - * - * \param group A value of type ::psa_dh_family_t that identifies the - * Diffie-Hellman group to be used. - */ -#define PSA_KEY_TYPE_DH_PUBLIC_KEY(group) \ - (PSA_KEY_TYPE_DH_PUBLIC_KEY_BASE | (group)) - -/** Whether a key type is a Diffie-Hellman key (pair or public-only). */ -#define PSA_KEY_TYPE_IS_DH(type) \ - ((PSA_KEY_TYPE_PUBLIC_KEY_OF_KEY_PAIR(type) & \ - ~PSA_KEY_TYPE_DH_GROUP_MASK) == PSA_KEY_TYPE_DH_PUBLIC_KEY_BASE) -/** Whether a key type is a Diffie-Hellman key pair. */ -#define PSA_KEY_TYPE_IS_DH_KEY_PAIR(type) \ - (((type) & ~PSA_KEY_TYPE_DH_GROUP_MASK) == \ - PSA_KEY_TYPE_DH_KEY_PAIR_BASE) -/** Whether a key type is a Diffie-Hellman public key. */ -#define PSA_KEY_TYPE_IS_DH_PUBLIC_KEY(type) \ - (((type) & ~PSA_KEY_TYPE_DH_GROUP_MASK) == \ - PSA_KEY_TYPE_DH_PUBLIC_KEY_BASE) - -/** Extract the group from a Diffie-Hellman key type. */ -#define PSA_KEY_TYPE_DH_GET_FAMILY(type) \ - ((psa_dh_family_t) (PSA_KEY_TYPE_IS_DH(type) ? \ - ((type) & PSA_KEY_TYPE_DH_GROUP_MASK) : \ - 0)) - -/** Diffie-Hellman groups defined in RFC 7919 Appendix A. - * - * This family includes groups with the following key sizes (in bits): - * 2048, 3072, 4096, 6144, 8192. A given implementation may support - * all of these sizes or only a subset. - */ -#define PSA_DH_FAMILY_RFC7919 ((psa_dh_family_t) 0x03) - -#define PSA_GET_KEY_TYPE_BLOCK_SIZE_EXPONENT(type) \ - (((type) >> 8) & 7) -/** The block size of a block cipher. - * - * \param type A cipher key type (value of type #psa_key_type_t). - * - * \return The block size for a block cipher, or 1 for a stream cipher. - * The return value is undefined if \p type is not a supported - * cipher key type. - * - * \note It is possible to build stream cipher algorithms on top of a block - * cipher, for example CTR mode (#PSA_ALG_CTR). - * This macro only takes the key type into account, so it cannot be - * used to determine the size of the data that #psa_cipher_update() - * might buffer for future processing in general. - * - * \note This macro returns a compile-time constant if its argument is one. - * - * \warning This macro may evaluate its argument multiple times. - */ -#define PSA_BLOCK_CIPHER_BLOCK_LENGTH(type) \ - (((type) & PSA_KEY_TYPE_CATEGORY_MASK) == PSA_KEY_TYPE_CATEGORY_SYMMETRIC ? \ - 1u << PSA_GET_KEY_TYPE_BLOCK_SIZE_EXPONENT(type) : \ - 0u) - -/* Note that algorithm values are embedded in the persistent key store, - * as part of key metadata. As a consequence, they must not be changed - * (unless the storage format version changes). - */ - -/** Vendor-defined algorithm flag. - * - * Algorithms defined by this standard will never have the #PSA_ALG_VENDOR_FLAG - * bit set. Vendors who define additional algorithms must use an encoding with - * the #PSA_ALG_VENDOR_FLAG bit set and should respect the bitwise structure - * used by standard encodings whenever practical. - */ -#define PSA_ALG_VENDOR_FLAG ((psa_algorithm_t) 0x80000000) - -#define PSA_ALG_CATEGORY_MASK ((psa_algorithm_t) 0x7f000000) -#define PSA_ALG_CATEGORY_HASH ((psa_algorithm_t) 0x02000000) -#define PSA_ALG_CATEGORY_MAC ((psa_algorithm_t) 0x03000000) -#define PSA_ALG_CATEGORY_CIPHER ((psa_algorithm_t) 0x04000000) -#define PSA_ALG_CATEGORY_AEAD ((psa_algorithm_t) 0x05000000) -#define PSA_ALG_CATEGORY_SIGN ((psa_algorithm_t) 0x06000000) -#define PSA_ALG_CATEGORY_ASYMMETRIC_ENCRYPTION ((psa_algorithm_t) 0x07000000) -#define PSA_ALG_CATEGORY_KEY_DERIVATION ((psa_algorithm_t) 0x08000000) -#define PSA_ALG_CATEGORY_KEY_AGREEMENT ((psa_algorithm_t) 0x09000000) - -/** Whether an algorithm is vendor-defined. - * - * See also #PSA_ALG_VENDOR_FLAG. - */ -#define PSA_ALG_IS_VENDOR_DEFINED(alg) \ - (((alg) & PSA_ALG_VENDOR_FLAG) != 0) - -/** Whether the specified algorithm is a hash algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a hash algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_HASH(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_HASH) - -/** Whether the specified algorithm is a MAC algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a MAC algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_MAC(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_MAC) - -/** Whether the specified algorithm is a symmetric cipher algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a symmetric cipher algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_CIPHER(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_CIPHER) - -/** Whether the specified algorithm is an authenticated encryption - * with associated data (AEAD) algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is an AEAD algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_AEAD(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_AEAD) - -/** Whether the specified algorithm is an asymmetric signature algorithm, - * also known as public-key signature algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is an asymmetric signature algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_SIGN(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_SIGN) - -/** Whether the specified algorithm is an asymmetric encryption algorithm, - * also known as public-key encryption algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is an asymmetric encryption algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_ASYMMETRIC_ENCRYPTION(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_ASYMMETRIC_ENCRYPTION) - -/** Whether the specified algorithm is a key agreement algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a key agreement algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_KEY_AGREEMENT(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_KEY_AGREEMENT) - -/** Whether the specified algorithm is a key derivation algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a key derivation algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_KEY_DERIVATION(alg) \ - (((alg) & PSA_ALG_CATEGORY_MASK) == PSA_ALG_CATEGORY_KEY_DERIVATION) - -/** Whether the specified algorithm is a key stretching / password hashing - * algorithm. - * - * A key stretching / password hashing algorithm is a key derivation algorithm - * that is suitable for use with a low-entropy secret such as a password. - * Equivalently, it's a key derivation algorithm that uses a - * #PSA_KEY_DERIVATION_INPUT_PASSWORD input step. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a key stretching / password hashing algorithm, 0 - * otherwise. This macro may return either 0 or 1 if \p alg is not a - * supported algorithm identifier. - */ -#define PSA_ALG_IS_KEY_DERIVATION_STRETCHING(alg) \ - (PSA_ALG_IS_KEY_DERIVATION(alg) && \ - (alg) & PSA_ALG_KEY_DERIVATION_STRETCHING_FLAG) - -/** An invalid algorithm identifier value. */ -/* *INDENT-OFF* (https://github.com/ARM-software/psa-arch-tests/issues/337) */ -#define PSA_ALG_NONE ((psa_algorithm_t)0) -/* *INDENT-ON* */ - -#define PSA_ALG_HASH_MASK ((psa_algorithm_t) 0x000000ff) -/** MD5 */ -#define PSA_ALG_MD5 ((psa_algorithm_t) 0x02000003) -/** PSA_ALG_RIPEMD160 */ -#define PSA_ALG_RIPEMD160 ((psa_algorithm_t) 0x02000004) -/** SHA1 */ -#define PSA_ALG_SHA_1 ((psa_algorithm_t) 0x02000005) -/** SHA2-224 */ -#define PSA_ALG_SHA_224 ((psa_algorithm_t) 0x02000008) -/** SHA2-256 */ -#define PSA_ALG_SHA_256 ((psa_algorithm_t) 0x02000009) -/** SHA2-384 */ -#define PSA_ALG_SHA_384 ((psa_algorithm_t) 0x0200000a) -/** SHA2-512 */ -#define PSA_ALG_SHA_512 ((psa_algorithm_t) 0x0200000b) -/** SHA2-512/224 */ -#define PSA_ALG_SHA_512_224 ((psa_algorithm_t) 0x0200000c) -/** SHA2-512/256 */ -#define PSA_ALG_SHA_512_256 ((psa_algorithm_t) 0x0200000d) -/** SHA3-224 */ -#define PSA_ALG_SHA3_224 ((psa_algorithm_t) 0x02000010) -/** SHA3-256 */ -#define PSA_ALG_SHA3_256 ((psa_algorithm_t) 0x02000011) -/** SHA3-384 */ -#define PSA_ALG_SHA3_384 ((psa_algorithm_t) 0x02000012) -/** SHA3-512 */ -#define PSA_ALG_SHA3_512 ((psa_algorithm_t) 0x02000013) -/** The first 512 bits (64 bytes) of the SHAKE256 output. - * - * This is the prehashing for Ed448ph (see #PSA_ALG_ED448PH). For other - * scenarios where a hash function based on SHA3/SHAKE is desired, SHA3-512 - * has the same output size and a (theoretically) higher security strength. - */ -#define PSA_ALG_SHAKE256_512 ((psa_algorithm_t) 0x02000015) - -/** In a hash-and-sign algorithm policy, allow any hash algorithm. - * - * This value may be used to form the algorithm usage field of a policy - * for a signature algorithm that is parametrized by a hash. The key - * may then be used to perform operations using the same signature - * algorithm parametrized with any supported hash. - * - * That is, suppose that `PSA_xxx_SIGNATURE` is one of the following macros: - * - #PSA_ALG_RSA_PKCS1V15_SIGN, #PSA_ALG_RSA_PSS, #PSA_ALG_RSA_PSS_ANY_SALT, - * - #PSA_ALG_ECDSA, #PSA_ALG_DETERMINISTIC_ECDSA. - * Then you may create and use a key as follows: - * - Set the key usage field using #PSA_ALG_ANY_HASH, for example: - * ``` - * psa_set_key_usage_flags(&attributes, PSA_KEY_USAGE_SIGN_HASH); // or VERIFY - * psa_set_key_algorithm(&attributes, PSA_xxx_SIGNATURE(PSA_ALG_ANY_HASH)); - * ``` - * - Import or generate key material. - * - Call psa_sign_hash() or psa_verify_hash(), passing - * an algorithm built from `PSA_xxx_SIGNATURE` and a specific hash. Each - * call to sign or verify a message may use a different hash. - * ``` - * psa_sign_hash(key, PSA_xxx_SIGNATURE(PSA_ALG_SHA_256), ...); - * psa_sign_hash(key, PSA_xxx_SIGNATURE(PSA_ALG_SHA_512), ...); - * psa_sign_hash(key, PSA_xxx_SIGNATURE(PSA_ALG_SHA3_256), ...); - * ``` - * - * This value may not be used to build other algorithms that are - * parametrized over a hash. For any valid use of this macro to build - * an algorithm \c alg, #PSA_ALG_IS_HASH_AND_SIGN(\c alg) is true. - * - * This value may not be used to build an algorithm specification to - * perform an operation. It is only valid to build policies. - */ -#define PSA_ALG_ANY_HASH ((psa_algorithm_t) 0x020000ff) - -#define PSA_ALG_MAC_SUBCATEGORY_MASK ((psa_algorithm_t) 0x00c00000) -#define PSA_ALG_HMAC_BASE ((psa_algorithm_t) 0x03800000) -/** Macro to build an HMAC algorithm. - * - * For example, #PSA_ALG_HMAC(#PSA_ALG_SHA_256) is HMAC-SHA-256. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding HMAC algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_HMAC(hash_alg) \ - (PSA_ALG_HMAC_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -#define PSA_ALG_HMAC_GET_HASH(hmac_alg) \ - (PSA_ALG_CATEGORY_HASH | ((hmac_alg) & PSA_ALG_HASH_MASK)) - -/** Whether the specified algorithm is an HMAC algorithm. - * - * HMAC is a family of MAC algorithms that are based on a hash function. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is an HMAC algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_HMAC(alg) \ - (((alg) & (PSA_ALG_CATEGORY_MASK | PSA_ALG_MAC_SUBCATEGORY_MASK)) == \ - PSA_ALG_HMAC_BASE) - -/* In the encoding of a MAC algorithm, the bits corresponding to - * PSA_ALG_MAC_TRUNCATION_MASK encode the length to which the MAC is - * truncated. As an exception, the value 0 means the untruncated algorithm, - * whatever its length is. The length is encoded in 6 bits, so it can - * reach up to 63; the largest MAC is 64 bytes so its trivial truncation - * to full length is correctly encoded as 0 and any non-trivial truncation - * is correctly encoded as a value between 1 and 63. */ -#define PSA_ALG_MAC_TRUNCATION_MASK ((psa_algorithm_t) 0x003f0000) -#define PSA_MAC_TRUNCATION_OFFSET 16 - -/* In the encoding of a MAC algorithm, the bit corresponding to - * #PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG encodes the fact that the algorithm - * is a wildcard algorithm. A key with such wildcard algorithm as permitted - * algorithm policy can be used with any algorithm corresponding to the - * same base class and having a (potentially truncated) MAC length greater or - * equal than the one encoded in #PSA_ALG_MAC_TRUNCATION_MASK. */ -#define PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG ((psa_algorithm_t) 0x00008000) - -/** Macro to build a truncated MAC algorithm. - * - * A truncated MAC algorithm is identical to the corresponding MAC - * algorithm except that the MAC value for the truncated algorithm - * consists of only the first \p mac_length bytes of the MAC value - * for the untruncated algorithm. - * - * \note This macro may allow constructing algorithm identifiers that - * are not valid, either because the specified length is larger - * than the untruncated MAC or because the specified length is - * smaller than permitted by the implementation. - * - * \note It is implementation-defined whether a truncated MAC that - * is truncated to the same length as the MAC of the untruncated - * algorithm is considered identical to the untruncated algorithm - * for policy comparison purposes. - * - * \param mac_alg A MAC algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p mac_alg) - * is true). This may be a truncated or untruncated - * MAC algorithm. - * \param mac_length Desired length of the truncated MAC in bytes. - * This must be at most the full length of the MAC - * and must be at least an implementation-specified - * minimum. The implementation-specified minimum - * shall not be zero. - * - * \return The corresponding MAC algorithm with the specified - * length. - * \return Unspecified if \p mac_alg is not a supported - * MAC algorithm or if \p mac_length is too small or - * too large for the specified MAC algorithm. - */ -#define PSA_ALG_TRUNCATED_MAC(mac_alg, mac_length) \ - (((mac_alg) & ~(PSA_ALG_MAC_TRUNCATION_MASK | \ - PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG)) | \ - ((mac_length) << PSA_MAC_TRUNCATION_OFFSET & PSA_ALG_MAC_TRUNCATION_MASK)) - -/** Macro to build the base MAC algorithm corresponding to a truncated - * MAC algorithm. - * - * \param mac_alg A MAC algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p mac_alg) - * is true). This may be a truncated or untruncated - * MAC algorithm. - * - * \return The corresponding base MAC algorithm. - * \return Unspecified if \p mac_alg is not a supported - * MAC algorithm. - */ -#define PSA_ALG_FULL_LENGTH_MAC(mac_alg) \ - ((mac_alg) & ~(PSA_ALG_MAC_TRUNCATION_MASK | \ - PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG)) - -/** Length to which a MAC algorithm is truncated. - * - * \param mac_alg A MAC algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p mac_alg) - * is true). - * - * \return Length of the truncated MAC in bytes. - * \return 0 if \p mac_alg is a non-truncated MAC algorithm. - * \return Unspecified if \p mac_alg is not a supported - * MAC algorithm. - */ -#define PSA_MAC_TRUNCATED_LENGTH(mac_alg) \ - (((mac_alg) & PSA_ALG_MAC_TRUNCATION_MASK) >> PSA_MAC_TRUNCATION_OFFSET) - -/** Macro to build a MAC minimum-MAC-length wildcard algorithm. - * - * A minimum-MAC-length MAC wildcard algorithm permits all MAC algorithms - * sharing the same base algorithm, and where the (potentially truncated) MAC - * length of the specific algorithm is equal to or larger then the wildcard - * algorithm's minimum MAC length. - * - * \note When setting the minimum required MAC length to less than the - * smallest MAC length allowed by the base algorithm, this effectively - * becomes an 'any-MAC-length-allowed' policy for that base algorithm. - * - * \param mac_alg A MAC algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_MAC(\p mac_alg) - * is true). - * \param min_mac_length Desired minimum length of the message authentication - * code in bytes. This must be at most the untruncated - * length of the MAC and must be at least 1. - * - * \return The corresponding MAC wildcard algorithm with the - * specified minimum length. - * \return Unspecified if \p mac_alg is not a supported MAC - * algorithm or if \p min_mac_length is less than 1 or - * too large for the specified MAC algorithm. - */ -#define PSA_ALG_AT_LEAST_THIS_LENGTH_MAC(mac_alg, min_mac_length) \ - (PSA_ALG_TRUNCATED_MAC(mac_alg, min_mac_length) | \ - PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG) - -#define PSA_ALG_CIPHER_MAC_BASE ((psa_algorithm_t) 0x03c00000) -/** The CBC-MAC construction over a block cipher - * - * \warning CBC-MAC is insecure in many cases. - * A more secure mode, such as #PSA_ALG_CMAC, is recommended. - */ -#define PSA_ALG_CBC_MAC ((psa_algorithm_t) 0x03c00100) -/** The CMAC construction over a block cipher */ -#define PSA_ALG_CMAC ((psa_algorithm_t) 0x03c00200) - -/** Whether the specified algorithm is a MAC algorithm based on a block cipher. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a MAC algorithm based on a block cipher, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_BLOCK_CIPHER_MAC(alg) \ - (((alg) & (PSA_ALG_CATEGORY_MASK | PSA_ALG_MAC_SUBCATEGORY_MASK)) == \ - PSA_ALG_CIPHER_MAC_BASE) - -#define PSA_ALG_CIPHER_STREAM_FLAG ((psa_algorithm_t) 0x00800000) -#define PSA_ALG_CIPHER_FROM_BLOCK_FLAG ((psa_algorithm_t) 0x00400000) - -/** Whether the specified algorithm is a stream cipher. - * - * A stream cipher is a symmetric cipher that encrypts or decrypts messages - * by applying a bitwise-xor with a stream of bytes that is generated - * from a key. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a stream cipher algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier or if it is not a symmetric cipher algorithm. - */ -#define PSA_ALG_IS_STREAM_CIPHER(alg) \ - (((alg) & (PSA_ALG_CATEGORY_MASK | PSA_ALG_CIPHER_STREAM_FLAG)) == \ - (PSA_ALG_CATEGORY_CIPHER | PSA_ALG_CIPHER_STREAM_FLAG)) - -/** The stream cipher mode of a stream cipher algorithm. - * - * The underlying stream cipher is determined by the key type. - * - To use ChaCha20, use a key type of #PSA_KEY_TYPE_CHACHA20. - */ -#define PSA_ALG_STREAM_CIPHER ((psa_algorithm_t) 0x04800100) - -/** The CTR stream cipher mode. - * - * CTR is a stream cipher which is built from a block cipher. - * The underlying block cipher is determined by the key type. - * For example, to use AES-128-CTR, use this algorithm with - * a key of type #PSA_KEY_TYPE_AES and a length of 128 bits (16 bytes). - */ -#define PSA_ALG_CTR ((psa_algorithm_t) 0x04c01000) - -/** The CFB stream cipher mode. - * - * The underlying block cipher is determined by the key type. - */ -#define PSA_ALG_CFB ((psa_algorithm_t) 0x04c01100) - -/** The OFB stream cipher mode. - * - * The underlying block cipher is determined by the key type. - */ -#define PSA_ALG_OFB ((psa_algorithm_t) 0x04c01200) - -/** The XTS cipher mode. - * - * XTS is a cipher mode which is built from a block cipher. It requires at - * least one full block of input, but beyond this minimum the input - * does not need to be a whole number of blocks. - */ -#define PSA_ALG_XTS ((psa_algorithm_t) 0x0440ff00) - -/** The Electronic Code Book (ECB) mode of a block cipher, with no padding. - * - * \warning ECB mode does not protect the confidentiality of the encrypted data - * except in extremely narrow circumstances. It is recommended that applications - * only use ECB if they need to construct an operating mode that the - * implementation does not provide. Implementations are encouraged to provide - * the modes that applications need in preference to supporting direct access - * to ECB. - * - * The underlying block cipher is determined by the key type. - * - * This symmetric cipher mode can only be used with messages whose lengths are a - * multiple of the block size of the chosen block cipher. - * - * ECB mode does not accept an initialization vector (IV). When using a - * multi-part cipher operation with this algorithm, psa_cipher_generate_iv() - * and psa_cipher_set_iv() must not be called. - */ -#define PSA_ALG_ECB_NO_PADDING ((psa_algorithm_t) 0x04404400) - -/** The CBC block cipher chaining mode, with no padding. - * - * The underlying block cipher is determined by the key type. - * - * This symmetric cipher mode can only be used with messages whose lengths - * are whole number of blocks for the chosen block cipher. - */ -#define PSA_ALG_CBC_NO_PADDING ((psa_algorithm_t) 0x04404000) - -/** The CBC block cipher chaining mode with PKCS#7 padding. - * - * The underlying block cipher is determined by the key type. - * - * This is the padding method defined by PKCS#7 (RFC 2315) §10.3. - */ -#define PSA_ALG_CBC_PKCS7 ((psa_algorithm_t) 0x04404100) - -#define PSA_ALG_AEAD_FROM_BLOCK_FLAG ((psa_algorithm_t) 0x00400000) - -/** Whether the specified algorithm is an AEAD mode on a block cipher. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is an AEAD algorithm which is an AEAD mode based on - * a block cipher, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_AEAD_ON_BLOCK_CIPHER(alg) \ - (((alg) & (PSA_ALG_CATEGORY_MASK | PSA_ALG_AEAD_FROM_BLOCK_FLAG)) == \ - (PSA_ALG_CATEGORY_AEAD | PSA_ALG_AEAD_FROM_BLOCK_FLAG)) - -/** The CCM authenticated encryption algorithm. - * - * The underlying block cipher is determined by the key type. - */ -#define PSA_ALG_CCM ((psa_algorithm_t) 0x05500100) - -/** The CCM* cipher mode without authentication. - * - * This is CCM* as specified in IEEE 802.15.4 §7, with a tag length of 0. - * For CCM* with a nonzero tag length, use the AEAD algorithm #PSA_ALG_CCM. - * - * The underlying block cipher is determined by the key type. - * - * Currently only 13-byte long IV's are supported. - */ -#define PSA_ALG_CCM_STAR_NO_TAG ((psa_algorithm_t) 0x04c01300) - -/** The GCM authenticated encryption algorithm. - * - * The underlying block cipher is determined by the key type. - */ -#define PSA_ALG_GCM ((psa_algorithm_t) 0x05500200) - -/** The Chacha20-Poly1305 AEAD algorithm. - * - * The ChaCha20_Poly1305 construction is defined in RFC 7539. - * - * Implementations must support 12-byte nonces, may support 8-byte nonces, - * and should reject other sizes. - * - * Implementations must support 16-byte tags and should reject other sizes. - */ -#define PSA_ALG_CHACHA20_POLY1305 ((psa_algorithm_t) 0x05100500) - -/* In the encoding of an AEAD algorithm, the bits corresponding to - * PSA_ALG_AEAD_TAG_LENGTH_MASK encode the length of the AEAD tag. - * The constants for default lengths follow this encoding. - */ -#define PSA_ALG_AEAD_TAG_LENGTH_MASK ((psa_algorithm_t) 0x003f0000) -#define PSA_AEAD_TAG_LENGTH_OFFSET 16 - -/* In the encoding of an AEAD algorithm, the bit corresponding to - * #PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG encodes the fact that the algorithm - * is a wildcard algorithm. A key with such wildcard algorithm as permitted - * algorithm policy can be used with any algorithm corresponding to the - * same base class and having a tag length greater than or equal to the one - * encoded in #PSA_ALG_AEAD_TAG_LENGTH_MASK. */ -#define PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG ((psa_algorithm_t) 0x00008000) - -/** Macro to build a shortened AEAD algorithm. - * - * A shortened AEAD algorithm is similar to the corresponding AEAD - * algorithm, but has an authentication tag that consists of fewer bytes. - * Depending on the algorithm, the tag length may affect the calculation - * of the ciphertext. - * - * \param aead_alg An AEAD algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_AEAD(\p aead_alg) - * is true). - * \param tag_length Desired length of the authentication tag in bytes. - * - * \return The corresponding AEAD algorithm with the specified - * length. - * \return Unspecified if \p aead_alg is not a supported - * AEAD algorithm or if \p tag_length is not valid - * for the specified AEAD algorithm. - */ -#define PSA_ALG_AEAD_WITH_SHORTENED_TAG(aead_alg, tag_length) \ - (((aead_alg) & ~(PSA_ALG_AEAD_TAG_LENGTH_MASK | \ - PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG)) | \ - ((tag_length) << PSA_AEAD_TAG_LENGTH_OFFSET & \ - PSA_ALG_AEAD_TAG_LENGTH_MASK)) - -/** Retrieve the tag length of a specified AEAD algorithm - * - * \param aead_alg An AEAD algorithm identifier (value of type - * #psa_algorithm_t such that #PSA_ALG_IS_AEAD(\p aead_alg) - * is true). - * - * \return The tag length specified by the input algorithm. - * \return Unspecified if \p aead_alg is not a supported - * AEAD algorithm. - */ -#define PSA_ALG_AEAD_GET_TAG_LENGTH(aead_alg) \ - (((aead_alg) & PSA_ALG_AEAD_TAG_LENGTH_MASK) >> \ - PSA_AEAD_TAG_LENGTH_OFFSET) - -/** Calculate the corresponding AEAD algorithm with the default tag length. - * - * \param aead_alg An AEAD algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_AEAD(\p aead_alg) is true). - * - * \return The corresponding AEAD algorithm with the default - * tag length for that algorithm. - */ -#define PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG(aead_alg) \ - ( \ - PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE(aead_alg, PSA_ALG_CCM) \ - PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE(aead_alg, PSA_ALG_GCM) \ - PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE(aead_alg, PSA_ALG_CHACHA20_POLY1305) \ - 0) -#define PSA_ALG_AEAD_WITH_DEFAULT_LENGTH_TAG_CASE(aead_alg, ref) \ - PSA_ALG_AEAD_WITH_SHORTENED_TAG(aead_alg, 0) == \ - PSA_ALG_AEAD_WITH_SHORTENED_TAG(ref, 0) ? \ - ref : - -/** Macro to build an AEAD minimum-tag-length wildcard algorithm. - * - * A minimum-tag-length AEAD wildcard algorithm permits all AEAD algorithms - * sharing the same base algorithm, and where the tag length of the specific - * algorithm is equal to or larger then the minimum tag length specified by the - * wildcard algorithm. - * - * \note When setting the minimum required tag length to less than the - * smallest tag length allowed by the base algorithm, this effectively - * becomes an 'any-tag-length-allowed' policy for that base algorithm. - * - * \param aead_alg An AEAD algorithm identifier (value of type - * #psa_algorithm_t such that - * #PSA_ALG_IS_AEAD(\p aead_alg) is true). - * \param min_tag_length Desired minimum length of the authentication tag in - * bytes. This must be at least 1 and at most the largest - * allowed tag length of the algorithm. - * - * \return The corresponding AEAD wildcard algorithm with the - * specified minimum length. - * \return Unspecified if \p aead_alg is not a supported - * AEAD algorithm or if \p min_tag_length is less than 1 - * or too large for the specified AEAD algorithm. - */ -#define PSA_ALG_AEAD_WITH_AT_LEAST_THIS_LENGTH_TAG(aead_alg, min_tag_length) \ - (PSA_ALG_AEAD_WITH_SHORTENED_TAG(aead_alg, min_tag_length) | \ - PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG) - -#define PSA_ALG_RSA_PKCS1V15_SIGN_BASE ((psa_algorithm_t) 0x06000200) -/** RSA PKCS#1 v1.5 signature with hashing. - * - * This is the signature scheme defined by RFC 8017 - * (PKCS#1: RSA Cryptography Specifications) under the name - * RSASSA-PKCS1-v1_5. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding RSA PKCS#1 v1.5 signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_RSA_PKCS1V15_SIGN(hash_alg) \ - (PSA_ALG_RSA_PKCS1V15_SIGN_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -/** Raw PKCS#1 v1.5 signature. - * - * The input to this algorithm is the DigestInfo structure used by - * RFC 8017 (PKCS#1: RSA Cryptography Specifications), §9.2 - * steps 3–6. - */ -#define PSA_ALG_RSA_PKCS1V15_SIGN_RAW PSA_ALG_RSA_PKCS1V15_SIGN_BASE -#define PSA_ALG_IS_RSA_PKCS1V15_SIGN(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_RSA_PKCS1V15_SIGN_BASE) - -#define PSA_ALG_RSA_PSS_BASE ((psa_algorithm_t) 0x06000300) -#define PSA_ALG_RSA_PSS_ANY_SALT_BASE ((psa_algorithm_t) 0x06001300) -/** RSA PSS signature with hashing. - * - * This is the signature scheme defined by RFC 8017 - * (PKCS#1: RSA Cryptography Specifications) under the name - * RSASSA-PSS, with the message generation function MGF1, and with - * a salt length equal to the length of the hash, or the largest - * possible salt length for the algorithm and key size if that is - * smaller than the hash length. The specified hash algorithm is - * used to hash the input message, to create the salted hash, and - * for the mask generation. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding RSA PSS signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_RSA_PSS(hash_alg) \ - (PSA_ALG_RSA_PSS_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -/** RSA PSS signature with hashing with relaxed verification. - * - * This algorithm has the same behavior as #PSA_ALG_RSA_PSS when signing, - * but allows an arbitrary salt length (including \c 0) when verifying a - * signature. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding RSA PSS signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_RSA_PSS_ANY_SALT(hash_alg) \ - (PSA_ALG_RSA_PSS_ANY_SALT_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -/** Whether the specified algorithm is RSA PSS with standard salt. - * - * \param alg An algorithm value or an algorithm policy wildcard. - * - * \return 1 if \p alg is of the form - * #PSA_ALG_RSA_PSS(\c hash_alg), - * where \c hash_alg is a hash algorithm or - * #PSA_ALG_ANY_HASH. 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not - * a supported algorithm identifier or policy. - */ -#define PSA_ALG_IS_RSA_PSS_STANDARD_SALT(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_RSA_PSS_BASE) - -/** Whether the specified algorithm is RSA PSS with any salt. - * - * \param alg An algorithm value or an algorithm policy wildcard. - * - * \return 1 if \p alg is of the form - * #PSA_ALG_RSA_PSS_ANY_SALT_BASE(\c hash_alg), - * where \c hash_alg is a hash algorithm or - * #PSA_ALG_ANY_HASH. 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not - * a supported algorithm identifier or policy. - */ -#define PSA_ALG_IS_RSA_PSS_ANY_SALT(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_RSA_PSS_ANY_SALT_BASE) - -/** Whether the specified algorithm is RSA PSS. - * - * This includes any of the RSA PSS algorithm variants, regardless of the - * constraints on salt length. - * - * \param alg An algorithm value or an algorithm policy wildcard. - * - * \return 1 if \p alg is of the form - * #PSA_ALG_RSA_PSS(\c hash_alg) or - * #PSA_ALG_RSA_PSS_ANY_SALT_BASE(\c hash_alg), - * where \c hash_alg is a hash algorithm or - * #PSA_ALG_ANY_HASH. 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not - * a supported algorithm identifier or policy. - */ -#define PSA_ALG_IS_RSA_PSS(alg) \ - (PSA_ALG_IS_RSA_PSS_STANDARD_SALT(alg) || \ - PSA_ALG_IS_RSA_PSS_ANY_SALT(alg)) - -#define PSA_ALG_ECDSA_BASE ((psa_algorithm_t) 0x06000600) -/** ECDSA signature with hashing. - * - * This is the ECDSA signature scheme defined by ANSI X9.62, - * with a random per-message secret number (*k*). - * - * The representation of the signature as a byte string consists of - * the concatenation of the signature values *r* and *s*. Each of - * *r* and *s* is encoded as an *N*-octet string, where *N* is the length - * of the base point of the curve in octets. Each value is represented - * in big-endian order (most significant octet first). - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding ECDSA signature algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_ECDSA(hash_alg) \ - (PSA_ALG_ECDSA_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -/** ECDSA signature without hashing. - * - * This is the same signature scheme as #PSA_ALG_ECDSA(), but - * without specifying a hash algorithm. This algorithm may only be - * used to sign or verify a sequence of bytes that should be an - * already-calculated hash. Note that the input is padded with - * zeros on the left or truncated on the left as required to fit - * the curve size. - */ -#define PSA_ALG_ECDSA_ANY PSA_ALG_ECDSA_BASE -#define PSA_ALG_DETERMINISTIC_ECDSA_BASE ((psa_algorithm_t) 0x06000700) -/** Deterministic ECDSA signature with hashing. - * - * This is the deterministic ECDSA signature scheme defined by RFC 6979. - * - * The representation of a signature is the same as with #PSA_ALG_ECDSA(). - * - * Note that when this algorithm is used for verification, signatures - * made with randomized ECDSA (#PSA_ALG_ECDSA(\p hash_alg)) with the - * same private key are accepted. In other words, - * #PSA_ALG_DETERMINISTIC_ECDSA(\p hash_alg) differs from - * #PSA_ALG_ECDSA(\p hash_alg) only for signature, not for verification. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * This includes #PSA_ALG_ANY_HASH - * when specifying the algorithm in a usage policy. - * - * \return The corresponding deterministic ECDSA signature - * algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_DETERMINISTIC_ECDSA(hash_alg) \ - (PSA_ALG_DETERMINISTIC_ECDSA_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -#define PSA_ALG_ECDSA_DETERMINISTIC_FLAG ((psa_algorithm_t) 0x00000100) -#define PSA_ALG_IS_ECDSA(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK & ~PSA_ALG_ECDSA_DETERMINISTIC_FLAG) == \ - PSA_ALG_ECDSA_BASE) -#define PSA_ALG_ECDSA_IS_DETERMINISTIC(alg) \ - (((alg) & PSA_ALG_ECDSA_DETERMINISTIC_FLAG) != 0) -#define PSA_ALG_IS_DETERMINISTIC_ECDSA(alg) \ - (PSA_ALG_IS_ECDSA(alg) && PSA_ALG_ECDSA_IS_DETERMINISTIC(alg)) -#define PSA_ALG_IS_RANDOMIZED_ECDSA(alg) \ - (PSA_ALG_IS_ECDSA(alg) && !PSA_ALG_ECDSA_IS_DETERMINISTIC(alg)) - -/** Edwards-curve digital signature algorithm without prehashing (PureEdDSA), - * using standard parameters. - * - * Contexts are not supported in the current version of this specification - * because there is no suitable signature interface that can take the - * context as a parameter. A future version of this specification may add - * suitable functions and extend this algorithm to support contexts. - * - * PureEdDSA requires an elliptic curve key on a twisted Edwards curve. - * In this specification, the following curves are supported: - * - #PSA_ECC_FAMILY_TWISTED_EDWARDS, 255-bit: Ed25519 as specified - * in RFC 8032. - * The curve is Edwards25519. - * The hash function used internally is SHA-512. - * - #PSA_ECC_FAMILY_TWISTED_EDWARDS, 448-bit: Ed448 as specified - * in RFC 8032. - * The curve is Edwards448. - * The hash function used internally is the first 114 bytes of the - * SHAKE256 output. - * - * This algorithm can be used with psa_sign_message() and - * psa_verify_message(). Since there is no prehashing, it cannot be used - * with psa_sign_hash() or psa_verify_hash(). - * - * The signature format is the concatenation of R and S as defined by - * RFC 8032 §5.1.6 and §5.2.6 (a 64-byte string for Ed25519, a 114-byte - * string for Ed448). - */ -#define PSA_ALG_PURE_EDDSA ((psa_algorithm_t) 0x06000800) - -#define PSA_ALG_HASH_EDDSA_BASE ((psa_algorithm_t) 0x06000900) -#define PSA_ALG_IS_HASH_EDDSA(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HASH_EDDSA_BASE) - -/** Edwards-curve digital signature algorithm with prehashing (HashEdDSA), - * using SHA-512 and the Edwards25519 curve. - * - * See #PSA_ALG_PURE_EDDSA regarding context support and the signature format. - * - * This algorithm is Ed25519 as specified in RFC 8032. - * The curve is Edwards25519. - * The prehash is SHA-512. - * The hash function used internally is SHA-512. - * - * This is a hash-and-sign algorithm: to calculate a signature, - * you can either: - * - call psa_sign_message() on the message; - * - or calculate the SHA-512 hash of the message - * with psa_hash_compute() - * or with a multi-part hash operation started with psa_hash_setup(), - * using the hash algorithm #PSA_ALG_SHA_512, - * then sign the calculated hash with psa_sign_hash(). - * Verifying a signature is similar, using psa_verify_message() or - * psa_verify_hash() instead of the signature function. - */ -#define PSA_ALG_ED25519PH \ - (PSA_ALG_HASH_EDDSA_BASE | (PSA_ALG_SHA_512 & PSA_ALG_HASH_MASK)) - -/** Edwards-curve digital signature algorithm with prehashing (HashEdDSA), - * using SHAKE256 and the Edwards448 curve. - * - * See #PSA_ALG_PURE_EDDSA regarding context support and the signature format. - * - * This algorithm is Ed448 as specified in RFC 8032. - * The curve is Edwards448. - * The prehash is the first 64 bytes of the SHAKE256 output. - * The hash function used internally is the first 114 bytes of the - * SHAKE256 output. - * - * This is a hash-and-sign algorithm: to calculate a signature, - * you can either: - * - call psa_sign_message() on the message; - * - or calculate the first 64 bytes of the SHAKE256 output of the message - * with psa_hash_compute() - * or with a multi-part hash operation started with psa_hash_setup(), - * using the hash algorithm #PSA_ALG_SHAKE256_512, - * then sign the calculated hash with psa_sign_hash(). - * Verifying a signature is similar, using psa_verify_message() or - * psa_verify_hash() instead of the signature function. - */ -#define PSA_ALG_ED448PH \ - (PSA_ALG_HASH_EDDSA_BASE | (PSA_ALG_SHAKE256_512 & PSA_ALG_HASH_MASK)) - -/* Default definition, to be overridden if the library is extended with - * more hash-and-sign algorithms that we want to keep out of this header - * file. */ -#define PSA_ALG_IS_VENDOR_HASH_AND_SIGN(alg) 0 - -/** Whether the specified algorithm is a signature algorithm that can be used - * with psa_sign_hash() and psa_verify_hash(). - * - * This encompasses all strict hash-and-sign algorithms categorized by - * PSA_ALG_IS_HASH_AND_SIGN(), as well as algorithms that follow the - * paradigm more loosely: - * - #PSA_ALG_RSA_PKCS1V15_SIGN_RAW (expects its input to be an encoded hash) - * - #PSA_ALG_ECDSA_ANY (doesn't specify what kind of hash the input is) - * - * \param alg An algorithm identifier (value of type psa_algorithm_t). - * - * \return 1 if alg is a signature algorithm that can be used to sign a - * hash. 0 if alg is a signature algorithm that can only be used - * to sign a message. 0 if alg is not a signature algorithm. - * This macro can return either 0 or 1 if alg is not a - * supported algorithm identifier. - */ -#define PSA_ALG_IS_SIGN_HASH(alg) \ - (PSA_ALG_IS_RSA_PSS(alg) || PSA_ALG_IS_RSA_PKCS1V15_SIGN(alg) || \ - PSA_ALG_IS_ECDSA(alg) || PSA_ALG_IS_HASH_EDDSA(alg) || \ - PSA_ALG_IS_VENDOR_HASH_AND_SIGN(alg)) - -/** Whether the specified algorithm is a signature algorithm that can be used - * with psa_sign_message() and psa_verify_message(). - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if alg is a signature algorithm that can be used to sign a - * message. 0 if \p alg is a signature algorithm that can only be used - * to sign an already-calculated hash. 0 if \p alg is not a signature - * algorithm. This macro can return either 0 or 1 if \p alg is not a - * supported algorithm identifier. - */ -#define PSA_ALG_IS_SIGN_MESSAGE(alg) \ - (PSA_ALG_IS_SIGN_HASH(alg) || (alg) == PSA_ALG_PURE_EDDSA) - -/** Whether the specified algorithm is a hash-and-sign algorithm. - * - * Hash-and-sign algorithms are asymmetric (public-key) signature algorithms - * structured in two parts: first the calculation of a hash in a way that - * does not depend on the key, then the calculation of a signature from the - * hash value and the key. Hash-and-sign algorithms encode the hash - * used for the hashing step, and you can call #PSA_ALG_SIGN_GET_HASH - * to extract this algorithm. - * - * Thus, for a hash-and-sign algorithm, - * `psa_sign_message(key, alg, input, ...)` is equivalent to - * ``` - * psa_hash_compute(PSA_ALG_SIGN_GET_HASH(alg), input, ..., hash, ...); - * psa_sign_hash(key, alg, hash, ..., signature, ...); - * ``` - * Most usefully, separating the hash from the signature allows the hash - * to be calculated in multiple steps with psa_hash_setup(), psa_hash_update() - * and psa_hash_finish(). Likewise psa_verify_message() is equivalent to - * calculating the hash and then calling psa_verify_hash(). - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a hash-and-sign algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_HASH_AND_SIGN(alg) \ - (PSA_ALG_IS_SIGN_HASH(alg) && \ - ((alg) & PSA_ALG_HASH_MASK) != 0) - -/** Get the hash used by a hash-and-sign signature algorithm. - * - * A hash-and-sign algorithm is a signature algorithm which is - * composed of two phases: first a hashing phase which does not use - * the key and produces a hash of the input message, then a signing - * phase which only uses the hash and the key and not the message - * itself. - * - * \param alg A signature algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_SIGN(\p alg) is true). - * - * \return The underlying hash algorithm if \p alg is a hash-and-sign - * algorithm. - * \return 0 if \p alg is a signature algorithm that does not - * follow the hash-and-sign structure. - * \return Unspecified if \p alg is not a signature algorithm or - * if it is not supported by the implementation. - */ -#define PSA_ALG_SIGN_GET_HASH(alg) \ - (PSA_ALG_IS_HASH_AND_SIGN(alg) ? \ - ((alg) & PSA_ALG_HASH_MASK) | PSA_ALG_CATEGORY_HASH : \ - 0) - -/** RSA PKCS#1 v1.5 encryption. - * - * \warning Calling psa_asymmetric_decrypt() with this algorithm as a - * parameter is considered an inherently dangerous function - * (CWE-242). Unless it is used in a side channel free and safe - * way (eg. implementing the TLS protocol as per 7.4.7.1 of - * RFC 5246), the calling code is vulnerable. - * - */ -#define PSA_ALG_RSA_PKCS1V15_CRYPT ((psa_algorithm_t) 0x07000200) - -#define PSA_ALG_RSA_OAEP_BASE ((psa_algorithm_t) 0x07000300) -/** RSA OAEP encryption. - * - * This is the encryption scheme defined by RFC 8017 - * (PKCS#1: RSA Cryptography Specifications) under the name - * RSAES-OAEP, with the message generation function MGF1. - * - * \param hash_alg The hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true) to use - * for MGF1. - * - * \return The corresponding RSA OAEP encryption algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_RSA_OAEP(hash_alg) \ - (PSA_ALG_RSA_OAEP_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -#define PSA_ALG_IS_RSA_OAEP(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_RSA_OAEP_BASE) -#define PSA_ALG_RSA_OAEP_GET_HASH(alg) \ - (PSA_ALG_IS_RSA_OAEP(alg) ? \ - ((alg) & PSA_ALG_HASH_MASK) | PSA_ALG_CATEGORY_HASH : \ - 0) - -#define PSA_ALG_HKDF_BASE ((psa_algorithm_t) 0x08000100) -/** Macro to build an HKDF algorithm. - * - * For example, `PSA_ALG_HKDF(PSA_ALG_SHA_256)` is HKDF using HMAC-SHA-256. - * - * This key derivation algorithm uses the following inputs: - * - #PSA_KEY_DERIVATION_INPUT_SALT is the salt used in the "extract" step. - * It is optional; if omitted, the derivation uses an empty salt. - * - #PSA_KEY_DERIVATION_INPUT_SECRET is the secret key used in the "extract" step. - * - #PSA_KEY_DERIVATION_INPUT_INFO is the info string used in the "expand" step. - * You must pass #PSA_KEY_DERIVATION_INPUT_SALT before #PSA_KEY_DERIVATION_INPUT_SECRET. - * You may pass #PSA_KEY_DERIVATION_INPUT_INFO at any time after steup and before - * starting to generate output. - * - * \warning HKDF processes the salt as follows: first hash it with hash_alg - * if the salt is longer than the block size of the hash algorithm; then - * pad with null bytes up to the block size. As a result, it is possible - * for distinct salt inputs to result in the same outputs. To ensure - * unique outputs, it is recommended to use a fixed length for salt values. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding HKDF algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_HKDF(hash_alg) \ - (PSA_ALG_HKDF_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -/** Whether the specified algorithm is an HKDF algorithm. - * - * HKDF is a family of key derivation algorithms that are based on a hash - * function and the HMAC construction. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is an HKDF algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_HKDF(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_BASE) -#define PSA_ALG_HKDF_GET_HASH(hkdf_alg) \ - (PSA_ALG_CATEGORY_HASH | ((hkdf_alg) & PSA_ALG_HASH_MASK)) - -#define PSA_ALG_HKDF_EXTRACT_BASE ((psa_algorithm_t) 0x08000400) -/** Macro to build an HKDF-Extract algorithm. - * - * For example, `PSA_ALG_HKDF_EXTRACT(PSA_ALG_SHA_256)` is - * HKDF-Extract using HMAC-SHA-256. - * - * This key derivation algorithm uses the following inputs: - * - PSA_KEY_DERIVATION_INPUT_SALT is the salt. - * - PSA_KEY_DERIVATION_INPUT_SECRET is the input keying material used in the - * "extract" step. - * The inputs are mandatory and must be passed in the order above. - * Each input may only be passed once. - * - * \warning HKDF-Extract is not meant to be used on its own. PSA_ALG_HKDF - * should be used instead if possible. PSA_ALG_HKDF_EXTRACT is provided - * as a separate algorithm for the sake of protocols that use it as a - * building block. It may also be a slight performance optimization - * in applications that use HKDF with the same salt and key but many - * different info strings. - * - * \warning HKDF processes the salt as follows: first hash it with hash_alg - * if the salt is longer than the block size of the hash algorithm; then - * pad with null bytes up to the block size. As a result, it is possible - * for distinct salt inputs to result in the same outputs. To ensure - * unique outputs, it is recommended to use a fixed length for salt values. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding HKDF-Extract algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_HKDF_EXTRACT(hash_alg) \ - (PSA_ALG_HKDF_EXTRACT_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -/** Whether the specified algorithm is an HKDF-Extract algorithm. - * - * HKDF-Extract is a family of key derivation algorithms that are based - * on a hash function and the HMAC construction. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is an HKDF-Extract algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_HKDF_EXTRACT(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_EXTRACT_BASE) - -#define PSA_ALG_HKDF_EXPAND_BASE ((psa_algorithm_t) 0x08000500) -/** Macro to build an HKDF-Expand algorithm. - * - * For example, `PSA_ALG_HKDF_EXPAND(PSA_ALG_SHA_256)` is - * HKDF-Expand using HMAC-SHA-256. - * - * This key derivation algorithm uses the following inputs: - * - PSA_KEY_DERIVATION_INPUT_SECRET is the pseudorandom key (PRK). - * - PSA_KEY_DERIVATION_INPUT_INFO is the info string. - * - * The inputs are mandatory and must be passed in the order above. - * Each input may only be passed once. - * - * \warning HKDF-Expand is not meant to be used on its own. `PSA_ALG_HKDF` - * should be used instead if possible. `PSA_ALG_HKDF_EXPAND` is provided as - * a separate algorithm for the sake of protocols that use it as a building - * block. It may also be a slight performance optimization in applications - * that use HKDF with the same salt and key but many different info strings. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding HKDF-Expand algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_HKDF_EXPAND(hash_alg) \ - (PSA_ALG_HKDF_EXPAND_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) -/** Whether the specified algorithm is an HKDF-Expand algorithm. - * - * HKDF-Expand is a family of key derivation algorithms that are based - * on a hash function and the HMAC construction. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is an HKDF-Expand algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_HKDF_EXPAND(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_EXPAND_BASE) - -/** Whether the specified algorithm is an HKDF or HKDF-Extract or - * HKDF-Expand algorithm. - * - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is any HKDF type algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_ANY_HKDF(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_BASE || \ - ((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_EXTRACT_BASE || \ - ((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_HKDF_EXPAND_BASE) - -#define PSA_ALG_TLS12_PRF_BASE ((psa_algorithm_t) 0x08000200) -/** Macro to build a TLS-1.2 PRF algorithm. - * - * TLS 1.2 uses a custom pseudorandom function (PRF) for key schedule, - * specified in Section 5 of RFC 5246. It is based on HMAC and can be - * used with either SHA-256 or SHA-384. - * - * This key derivation algorithm uses the following inputs, which must be - * passed in the order given here: - * - #PSA_KEY_DERIVATION_INPUT_SEED is the seed. - * - #PSA_KEY_DERIVATION_INPUT_SECRET is the secret key. - * - #PSA_KEY_DERIVATION_INPUT_LABEL is the label. - * - * For the application to TLS-1.2 key expansion, the seed is the - * concatenation of ServerHello.Random + ClientHello.Random, - * and the label is "key expansion". - * - * For example, `PSA_ALG_TLS12_PRF(PSA_ALG_SHA_256)` represents the - * TLS 1.2 PRF using HMAC-SHA-256. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding TLS-1.2 PRF algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_TLS12_PRF(hash_alg) \ - (PSA_ALG_TLS12_PRF_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -/** Whether the specified algorithm is a TLS-1.2 PRF algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is a TLS-1.2 PRF algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_TLS12_PRF(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_TLS12_PRF_BASE) -#define PSA_ALG_TLS12_PRF_GET_HASH(hkdf_alg) \ - (PSA_ALG_CATEGORY_HASH | ((hkdf_alg) & PSA_ALG_HASH_MASK)) - -#define PSA_ALG_TLS12_PSK_TO_MS_BASE ((psa_algorithm_t) 0x08000300) -/** Macro to build a TLS-1.2 PSK-to-MasterSecret algorithm. - * - * In a pure-PSK handshake in TLS 1.2, the master secret is derived - * from the PreSharedKey (PSK) through the application of padding - * (RFC 4279, Section 2) and the TLS-1.2 PRF (RFC 5246, Section 5). - * The latter is based on HMAC and can be used with either SHA-256 - * or SHA-384. - * - * This key derivation algorithm uses the following inputs, which must be - * passed in the order given here: - * - #PSA_KEY_DERIVATION_INPUT_SEED is the seed. - * - #PSA_KEY_DERIVATION_INPUT_OTHER_SECRET is the other secret for the - * computation of the premaster secret. This input is optional; - * if omitted, it defaults to a string of null bytes with the same length - * as the secret (PSK) input. - * - #PSA_KEY_DERIVATION_INPUT_SECRET is the secret key. - * - #PSA_KEY_DERIVATION_INPUT_LABEL is the label. - * - * For the application to TLS-1.2, the seed (which is - * forwarded to the TLS-1.2 PRF) is the concatenation of the - * ClientHello.Random + ServerHello.Random, - * the label is "master secret" or "extended master secret" and - * the other secret depends on the key exchange specified in the cipher suite: - * - for a plain PSK cipher suite (RFC 4279, Section 2), omit - * PSA_KEY_DERIVATION_INPUT_OTHER_SECRET - * - for a DHE-PSK (RFC 4279, Section 3) or ECDHE-PSK cipher suite - * (RFC 5489, Section 2), the other secret should be the output of the - * PSA_ALG_FFDH or PSA_ALG_ECDH key agreement performed with the peer. - * The recommended way to pass this input is to use a key derivation - * algorithm constructed as - * PSA_ALG_KEY_AGREEMENT(ka_alg, PSA_ALG_TLS12_PSK_TO_MS(hash_alg)) - * and to call psa_key_derivation_key_agreement(). Alternatively, - * this input may be an output of `psa_raw_key_agreement()` passed with - * psa_key_derivation_input_bytes(), or an equivalent input passed with - * psa_key_derivation_input_bytes() or psa_key_derivation_input_key(). - * - for a RSA-PSK cipher suite (RFC 4279, Section 4), the other secret - * should be the 48-byte client challenge (the PreMasterSecret of - * (RFC 5246, Section 7.4.7.1)) concatenation of the TLS version and - * a 46-byte random string chosen by the client. On the server, this is - * typically an output of psa_asymmetric_decrypt() using - * PSA_ALG_RSA_PKCS1V15_CRYPT, passed to the key derivation operation - * with `psa_key_derivation_input_bytes()`. - * - * For example, `PSA_ALG_TLS12_PSK_TO_MS(PSA_ALG_SHA_256)` represents the - * TLS-1.2 PSK to MasterSecret derivation PRF using HMAC-SHA-256. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding TLS-1.2 PSK to MS algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_TLS12_PSK_TO_MS(hash_alg) \ - (PSA_ALG_TLS12_PSK_TO_MS_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -/** Whether the specified algorithm is a TLS-1.2 PSK to MS algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is a TLS-1.2 PSK to MS algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_TLS12_PSK_TO_MS(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_TLS12_PSK_TO_MS_BASE) -#define PSA_ALG_TLS12_PSK_TO_MS_GET_HASH(hkdf_alg) \ - (PSA_ALG_CATEGORY_HASH | ((hkdf_alg) & PSA_ALG_HASH_MASK)) - -/* The TLS 1.2 ECJPAKE-to-PMS KDF. It takes the shared secret K (an EC point - * in case of EC J-PAKE) and calculates SHA256(K.X) that the rest of TLS 1.2 - * will use to derive the session secret, as defined by step 2 of - * https://datatracker.ietf.org/doc/html/draft-cragie-tls-ecjpake-01#section-8.7. - * Uses PSA_ALG_SHA_256. - * This function takes a single input: - * #PSA_KEY_DERIVATION_INPUT_SECRET is the shared secret K from EC J-PAKE. - * The only supported curve is secp256r1 (the 256-bit curve in - * #PSA_ECC_FAMILY_SECP_R1), so the input must be exactly 65 bytes. - * The output has to be read as a single chunk of 32 bytes, defined as - * PSA_TLS12_ECJPAKE_TO_PMS_DATA_SIZE. - */ -#define PSA_ALG_TLS12_ECJPAKE_TO_PMS ((psa_algorithm_t) 0x08000609) - -/* This flag indicates whether the key derivation algorithm is suitable for - * use on low-entropy secrets such as password - these algorithms are also - * known as key stretching or password hashing schemes. These are also the - * algorithms that accepts inputs of type #PSA_KEY_DERIVATION_INPUT_PASSWORD. - * - * Those algorithms cannot be combined with a key agreement algorithm. - */ -#define PSA_ALG_KEY_DERIVATION_STRETCHING_FLAG ((psa_algorithm_t) 0x00800000) - -#define PSA_ALG_PBKDF2_HMAC_BASE ((psa_algorithm_t) 0x08800100) -/** Macro to build a PBKDF2-HMAC password hashing / key stretching algorithm. - * - * PBKDF2 is defined by PKCS#5, republished as RFC 8018 (section 5.2). - * This macro specifies the PBKDF2 algorithm constructed using a PRF based on - * HMAC with the specified hash. - * For example, `PSA_ALG_PBKDF2_HMAC(PSA_ALG_SHA_256)` specifies PBKDF2 - * using the PRF HMAC-SHA-256. - * - * This key derivation algorithm uses the following inputs, which must be - * provided in the following order: - * - #PSA_KEY_DERIVATION_INPUT_COST is the iteration count. - * This input step must be used exactly once. - * - #PSA_KEY_DERIVATION_INPUT_SALT is the salt. - * This input step must be used one or more times; if used several times, the - * inputs will be concatenated. This can be used to build the final salt - * from multiple sources, both public and secret (also known as pepper). - * - #PSA_KEY_DERIVATION_INPUT_PASSWORD is the password to be hashed. - * This input step must be used exactly once. - * - * \param hash_alg A hash algorithm (\c PSA_ALG_XXX value such that - * #PSA_ALG_IS_HASH(\p hash_alg) is true). - * - * \return The corresponding PBKDF2-HMAC-XXX algorithm. - * \return Unspecified if \p hash_alg is not a supported - * hash algorithm. - */ -#define PSA_ALG_PBKDF2_HMAC(hash_alg) \ - (PSA_ALG_PBKDF2_HMAC_BASE | ((hash_alg) & PSA_ALG_HASH_MASK)) - -/** Whether the specified algorithm is a PBKDF2-HMAC algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is a PBKDF2-HMAC algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key derivation algorithm identifier. - */ -#define PSA_ALG_IS_PBKDF2_HMAC(alg) \ - (((alg) & ~PSA_ALG_HASH_MASK) == PSA_ALG_PBKDF2_HMAC_BASE) -#define PSA_ALG_PBKDF2_HMAC_GET_HASH(pbkdf2_alg) \ - (PSA_ALG_CATEGORY_HASH | ((pbkdf2_alg) & PSA_ALG_HASH_MASK)) -/** The PBKDF2-AES-CMAC-PRF-128 password hashing / key stretching algorithm. - * - * PBKDF2 is defined by PKCS#5, republished as RFC 8018 (section 5.2). - * This macro specifies the PBKDF2 algorithm constructed using the - * AES-CMAC-PRF-128 PRF specified by RFC 4615. - * - * This key derivation algorithm uses the same inputs as - * #PSA_ALG_PBKDF2_HMAC() with the same constraints. - */ -#define PSA_ALG_PBKDF2_AES_CMAC_PRF_128 ((psa_algorithm_t) 0x08800200) - -#define PSA_ALG_IS_PBKDF2(kdf_alg) \ - (PSA_ALG_IS_PBKDF2_HMAC(kdf_alg) || \ - ((kdf_alg) == PSA_ALG_PBKDF2_AES_CMAC_PRF_128)) - -#define PSA_ALG_KEY_DERIVATION_MASK ((psa_algorithm_t) 0xfe00ffff) -#define PSA_ALG_KEY_AGREEMENT_MASK ((psa_algorithm_t) 0xffff0000) - -/** Macro to build a combined algorithm that chains a key agreement with - * a key derivation. - * - * \param ka_alg A key agreement algorithm (\c PSA_ALG_XXX value such - * that #PSA_ALG_IS_KEY_AGREEMENT(\p ka_alg) is true). - * \param kdf_alg A key derivation algorithm (\c PSA_ALG_XXX value such - * that #PSA_ALG_IS_KEY_DERIVATION(\p kdf_alg) is true). - * - * \return The corresponding key agreement and derivation - * algorithm. - * \return Unspecified if \p ka_alg is not a supported - * key agreement algorithm or \p kdf_alg is not a - * supported key derivation algorithm. - */ -#define PSA_ALG_KEY_AGREEMENT(ka_alg, kdf_alg) \ - ((ka_alg) | (kdf_alg)) - -#define PSA_ALG_KEY_AGREEMENT_GET_KDF(alg) \ - (((alg) & PSA_ALG_KEY_DERIVATION_MASK) | PSA_ALG_CATEGORY_KEY_DERIVATION) - -#define PSA_ALG_KEY_AGREEMENT_GET_BASE(alg) \ - (((alg) & PSA_ALG_KEY_AGREEMENT_MASK) | PSA_ALG_CATEGORY_KEY_AGREEMENT) - -/** Whether the specified algorithm is a raw key agreement algorithm. - * - * A raw key agreement algorithm is one that does not specify - * a key derivation function. - * Usually, raw key agreement algorithms are constructed directly with - * a \c PSA_ALG_xxx macro while non-raw key agreement algorithms are - * constructed with #PSA_ALG_KEY_AGREEMENT(). - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \p alg is a raw key agreement algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \p alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_RAW_KEY_AGREEMENT(alg) \ - (PSA_ALG_IS_KEY_AGREEMENT(alg) && \ - PSA_ALG_KEY_AGREEMENT_GET_KDF(alg) == PSA_ALG_CATEGORY_KEY_DERIVATION) - -#define PSA_ALG_IS_KEY_DERIVATION_OR_AGREEMENT(alg) \ - ((PSA_ALG_IS_KEY_DERIVATION(alg) || PSA_ALG_IS_KEY_AGREEMENT(alg))) - -/** The finite-field Diffie-Hellman (DH) key agreement algorithm. - * - * The shared secret produced by key agreement is - * `g^{ab}` in big-endian format. - * It is `ceiling(m / 8)` bytes long where `m` is the size of the prime `p` - * in bits. - */ -#define PSA_ALG_FFDH ((psa_algorithm_t) 0x09010000) - -/** Whether the specified algorithm is a finite field Diffie-Hellman algorithm. - * - * This includes the raw finite field Diffie-Hellman algorithm as well as - * finite-field Diffie-Hellman followed by any supporter key derivation - * algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is a finite field Diffie-Hellman algorithm, 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key agreement algorithm identifier. - */ -#define PSA_ALG_IS_FFDH(alg) \ - (PSA_ALG_KEY_AGREEMENT_GET_BASE(alg) == PSA_ALG_FFDH) - -/** The elliptic curve Diffie-Hellman (ECDH) key agreement algorithm. - * - * The shared secret produced by key agreement is the x-coordinate of - * the shared secret point. It is always `ceiling(m / 8)` bytes long where - * `m` is the bit size associated with the curve, i.e. the bit size of the - * order of the curve's coordinate field. When `m` is not a multiple of 8, - * the byte containing the most significant bit of the shared secret - * is padded with zero bits. The byte order is either little-endian - * or big-endian depending on the curve type. - * - * - For Montgomery curves (curve types `PSA_ECC_FAMILY_CURVEXXX`), - * the shared secret is the x-coordinate of `d_A Q_B = d_B Q_A` - * in little-endian byte order. - * The bit size is 448 for Curve448 and 255 for Curve25519. - * - For Weierstrass curves over prime fields (curve types - * `PSA_ECC_FAMILY_SECPXXX` and `PSA_ECC_FAMILY_BRAINPOOL_PXXX`), - * the shared secret is the x-coordinate of `d_A Q_B = d_B Q_A` - * in big-endian byte order. - * The bit size is `m = ceiling(log_2(p))` for the field `F_p`. - * - For Weierstrass curves over binary fields (curve types - * `PSA_ECC_FAMILY_SECTXXX`), - * the shared secret is the x-coordinate of `d_A Q_B = d_B Q_A` - * in big-endian byte order. - * The bit size is `m` for the field `F_{2^m}`. - */ -#define PSA_ALG_ECDH ((psa_algorithm_t) 0x09020000) - -/** Whether the specified algorithm is an elliptic curve Diffie-Hellman - * algorithm. - * - * This includes the raw elliptic curve Diffie-Hellman algorithm as well as - * elliptic curve Diffie-Hellman followed by any supporter key derivation - * algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is an elliptic curve Diffie-Hellman algorithm, - * 0 otherwise. - * This macro may return either 0 or 1 if \c alg is not a supported - * key agreement algorithm identifier. - */ -#define PSA_ALG_IS_ECDH(alg) \ - (PSA_ALG_KEY_AGREEMENT_GET_BASE(alg) == PSA_ALG_ECDH) - -/** Whether the specified algorithm encoding is a wildcard. - * - * Wildcard values may only be used to set the usage algorithm field in - * a policy, not to perform an operation. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return 1 if \c alg is a wildcard algorithm encoding. - * \return 0 if \c alg is a non-wildcard algorithm encoding (suitable for - * an operation). - * \return This macro may return either 0 or 1 if \c alg is not a supported - * algorithm identifier. - */ -#define PSA_ALG_IS_WILDCARD(alg) \ - (PSA_ALG_IS_HASH_AND_SIGN(alg) ? \ - PSA_ALG_SIGN_GET_HASH(alg) == PSA_ALG_ANY_HASH : \ - PSA_ALG_IS_MAC(alg) ? \ - (alg & PSA_ALG_MAC_AT_LEAST_THIS_LENGTH_FLAG) != 0 : \ - PSA_ALG_IS_AEAD(alg) ? \ - (alg & PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG) != 0 : \ - (alg) == PSA_ALG_ANY_HASH) - -/** Get the hash used by a composite algorithm. - * - * \param alg An algorithm identifier (value of type #psa_algorithm_t). - * - * \return The underlying hash algorithm if alg is a composite algorithm that - * uses a hash algorithm. - * - * \return \c 0 if alg is not a composite algorithm that uses a hash. - */ -#define PSA_ALG_GET_HASH(alg) \ - (((alg) & 0x000000ff) == 0 ? ((psa_algorithm_t) 0) : 0x02000000 | ((alg) & 0x000000ff)) - -/**@}*/ - -/** \defgroup key_lifetimes Key lifetimes - * @{ - */ - -/* Note that location and persistence level values are embedded in the - * persistent key store, as part of key metadata. As a consequence, they - * must not be changed (unless the storage format version changes). - */ - -/** The default lifetime for volatile keys. - * - * A volatile key only exists as long as the identifier to it is not destroyed. - * The key material is guaranteed to be erased on a power reset. - * - * A key with this lifetime is typically stored in the RAM area of the - * PSA Crypto subsystem. However this is an implementation choice. - * If an implementation stores data about the key in a non-volatile memory, - * it must release all the resources associated with the key and erase the - * key material if the calling application terminates. - */ -#define PSA_KEY_LIFETIME_VOLATILE ((psa_key_lifetime_t) 0x00000000) - -/** The default lifetime for persistent keys. - * - * A persistent key remains in storage until it is explicitly destroyed or - * until the corresponding storage area is wiped. This specification does - * not define any mechanism to wipe a storage area, but integrations may - * provide their own mechanism (for example to perform a factory reset, - * to prepare for device refurbishment, or to uninstall an application). - * - * This lifetime value is the default storage area for the calling - * application. Integrations of Mbed TLS may support other persistent lifetimes. - * See ::psa_key_lifetime_t for more information. - */ -#define PSA_KEY_LIFETIME_PERSISTENT ((psa_key_lifetime_t) 0x00000001) - -/** The persistence level of volatile keys. - * - * See ::psa_key_persistence_t for more information. - */ -#define PSA_KEY_PERSISTENCE_VOLATILE ((psa_key_persistence_t) 0x00) - -/** The default persistence level for persistent keys. - * - * See ::psa_key_persistence_t for more information. - */ -#define PSA_KEY_PERSISTENCE_DEFAULT ((psa_key_persistence_t) 0x01) - -/** A persistence level indicating that a key is never destroyed. - * - * See ::psa_key_persistence_t for more information. - */ -#define PSA_KEY_PERSISTENCE_READ_ONLY ((psa_key_persistence_t) 0xff) - -#define PSA_KEY_LIFETIME_GET_PERSISTENCE(lifetime) \ - ((psa_key_persistence_t) ((lifetime) & 0x000000ff)) - -#define PSA_KEY_LIFETIME_GET_LOCATION(lifetime) \ - ((psa_key_location_t) ((lifetime) >> 8)) - -/** Whether a key lifetime indicates that the key is volatile. - * - * A volatile key is automatically destroyed by the implementation when - * the application instance terminates. In particular, a volatile key - * is automatically destroyed on a power reset of the device. - * - * A key that is not volatile is persistent. Persistent keys are - * preserved until the application explicitly destroys them or until an - * implementation-specific device management event occurs (for example, - * a factory reset). - * - * \param lifetime The lifetime value to query (value of type - * ::psa_key_lifetime_t). - * - * \return \c 1 if the key is volatile, otherwise \c 0. - */ -#define PSA_KEY_LIFETIME_IS_VOLATILE(lifetime) \ - (PSA_KEY_LIFETIME_GET_PERSISTENCE(lifetime) == \ - PSA_KEY_PERSISTENCE_VOLATILE) - -/** Whether a key lifetime indicates that the key is read-only. - * - * Read-only keys cannot be created or destroyed through the PSA Crypto API. - * They must be created through platform-specific means that bypass the API. - * - * Some platforms may offer ways to destroy read-only keys. For example, - * consider a platform with multiple levels of privilege, where a - * low-privilege application can use a key but is not allowed to destroy - * it, and the platform exposes the key to the application with a read-only - * lifetime. High-privilege code can destroy the key even though the - * application sees the key as read-only. - * - * \param lifetime The lifetime value to query (value of type - * ::psa_key_lifetime_t). - * - * \return \c 1 if the key is read-only, otherwise \c 0. - */ -#define PSA_KEY_LIFETIME_IS_READ_ONLY(lifetime) \ - (PSA_KEY_LIFETIME_GET_PERSISTENCE(lifetime) == \ - PSA_KEY_PERSISTENCE_READ_ONLY) - -/** Construct a lifetime from a persistence level and a location. - * - * \param persistence The persistence level - * (value of type ::psa_key_persistence_t). - * \param location The location indicator - * (value of type ::psa_key_location_t). - * - * \return The constructed lifetime value. - */ -#define PSA_KEY_LIFETIME_FROM_PERSISTENCE_AND_LOCATION(persistence, location) \ - ((location) << 8 | (persistence)) - -/** The local storage area for persistent keys. - * - * This storage area is available on all systems that can store persistent - * keys without delegating the storage to a third-party cryptoprocessor. - * - * See ::psa_key_location_t for more information. - */ -#define PSA_KEY_LOCATION_LOCAL_STORAGE ((psa_key_location_t) 0x000000) - -#define PSA_KEY_LOCATION_VENDOR_FLAG ((psa_key_location_t) 0x800000) - -/* Note that key identifier values are embedded in the - * persistent key store, as part of key metadata. As a consequence, they - * must not be changed (unless the storage format version changes). - */ - -/** The null key identifier. - */ -/* *INDENT-OFF* (https://github.com/ARM-software/psa-arch-tests/issues/337) */ -#define PSA_KEY_ID_NULL ((psa_key_id_t)0) -/* *INDENT-ON* */ -/** The minimum value for a key identifier chosen by the application. - */ -#define PSA_KEY_ID_USER_MIN ((psa_key_id_t) 0x00000001) -/** The maximum value for a key identifier chosen by the application. - */ -#define PSA_KEY_ID_USER_MAX ((psa_key_id_t) 0x3fffffff) -/** The minimum value for a key identifier chosen by the implementation. - */ -#define PSA_KEY_ID_VENDOR_MIN ((psa_key_id_t) 0x40000000) -/** The maximum value for a key identifier chosen by the implementation. - */ -#define PSA_KEY_ID_VENDOR_MAX ((psa_key_id_t) 0x7fffffff) - - -#if !defined(MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER) - -#define MBEDTLS_SVC_KEY_ID_INIT ((psa_key_id_t) 0) -#define MBEDTLS_SVC_KEY_ID_GET_KEY_ID(id) (id) -#define MBEDTLS_SVC_KEY_ID_GET_OWNER_ID(id) (0) - -/** Utility to initialize a key identifier at runtime. - * - * \param unused Unused parameter. - * \param key_id Identifier of the key. - */ -static inline mbedtls_svc_key_id_t mbedtls_svc_key_id_make( - unsigned int unused, psa_key_id_t key_id) -{ - (void) unused; - - return key_id; -} - -/** Compare two key identifiers. - * - * \param id1 First key identifier. - * \param id2 Second key identifier. - * - * \return Non-zero if the two key identifier are equal, zero otherwise. - */ -static inline int mbedtls_svc_key_id_equal(mbedtls_svc_key_id_t id1, - mbedtls_svc_key_id_t id2) -{ - return id1 == id2; -} - -/** Check whether a key identifier is null. - * - * \param key Key identifier. - * - * \return Non-zero if the key identifier is null, zero otherwise. - */ -static inline int mbedtls_svc_key_id_is_null(mbedtls_svc_key_id_t key) -{ - return key == 0; -} - -#else /* MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER */ - -#define MBEDTLS_SVC_KEY_ID_INIT ((mbedtls_svc_key_id_t){ 0, 0 }) -#define MBEDTLS_SVC_KEY_ID_GET_KEY_ID(id) ((id).MBEDTLS_PRIVATE(key_id)) -#define MBEDTLS_SVC_KEY_ID_GET_OWNER_ID(id) ((id).MBEDTLS_PRIVATE(owner)) - -/** Utility to initialize a key identifier at runtime. - * - * \param owner_id Identifier of the key owner. - * \param key_id Identifier of the key. - */ -static inline mbedtls_svc_key_id_t mbedtls_svc_key_id_make( - mbedtls_key_owner_id_t owner_id, psa_key_id_t key_id) -{ - return (mbedtls_svc_key_id_t){ .MBEDTLS_PRIVATE(key_id) = key_id, - .MBEDTLS_PRIVATE(owner) = owner_id }; -} - -/** Compare two key identifiers. - * - * \param id1 First key identifier. - * \param id2 Second key identifier. - * - * \return Non-zero if the two key identifier are equal, zero otherwise. - */ -static inline int mbedtls_svc_key_id_equal(mbedtls_svc_key_id_t id1, - mbedtls_svc_key_id_t id2) -{ - return (id1.MBEDTLS_PRIVATE(key_id) == id2.MBEDTLS_PRIVATE(key_id)) && - mbedtls_key_owner_id_equal(id1.MBEDTLS_PRIVATE(owner), id2.MBEDTLS_PRIVATE(owner)); -} - -/** Check whether a key identifier is null. - * - * \param key Key identifier. - * - * \return Non-zero if the key identifier is null, zero otherwise. - */ -static inline int mbedtls_svc_key_id_is_null(mbedtls_svc_key_id_t key) -{ - return key.MBEDTLS_PRIVATE(key_id) == 0; -} - -#endif /* !MBEDTLS_PSA_CRYPTO_KEY_ID_ENCODES_OWNER */ - -/**@}*/ - -/** \defgroup policy Key policies - * @{ - */ - -/* Note that key usage flags are embedded in the - * persistent key store, as part of key metadata. As a consequence, they - * must not be changed (unless the storage format version changes). - */ - -/** Whether the key may be exported. - * - * A public key or the public part of a key pair may always be exported - * regardless of the value of this permission flag. - * - * If a key does not have export permission, implementations shall not - * allow the key to be exported in plain form from the cryptoprocessor, - * whether through psa_export_key() or through a proprietary interface. - * The key may however be exportable in a wrapped form, i.e. in a form - * where it is encrypted by another key. - */ -#define PSA_KEY_USAGE_EXPORT ((psa_key_usage_t) 0x00000001) - -/** Whether the key may be copied. - * - * This flag allows the use of psa_copy_key() to make a copy of the key - * with the same policy or a more restrictive policy. - * - * For lifetimes for which the key is located in a secure element which - * enforce the non-exportability of keys, copying a key outside the secure - * element also requires the usage flag #PSA_KEY_USAGE_EXPORT. - * Copying the key inside the secure element is permitted with just - * #PSA_KEY_USAGE_COPY if the secure element supports it. - * For keys with the lifetime #PSA_KEY_LIFETIME_VOLATILE or - * #PSA_KEY_LIFETIME_PERSISTENT, the usage flag #PSA_KEY_USAGE_COPY - * is sufficient to permit the copy. - */ -#define PSA_KEY_USAGE_COPY ((psa_key_usage_t) 0x00000002) - -/** Whether the key may be used to encrypt a message. - * - * This flag allows the key to be used for a symmetric encryption operation, - * for an AEAD encryption-and-authentication operation, - * or for an asymmetric encryption operation, - * if otherwise permitted by the key's type and policy. - * - * For a key pair, this concerns the public key. - */ -#define PSA_KEY_USAGE_ENCRYPT ((psa_key_usage_t) 0x00000100) - -/** Whether the key may be used to decrypt a message. - * - * This flag allows the key to be used for a symmetric decryption operation, - * for an AEAD decryption-and-verification operation, - * or for an asymmetric decryption operation, - * if otherwise permitted by the key's type and policy. - * - * For a key pair, this concerns the private key. - */ -#define PSA_KEY_USAGE_DECRYPT ((psa_key_usage_t) 0x00000200) - -/** Whether the key may be used to sign a message. - * - * This flag allows the key to be used for a MAC calculation operation or for - * an asymmetric message signature operation, if otherwise permitted by the - * key’s type and policy. - * - * For a key pair, this concerns the private key. - */ -#define PSA_KEY_USAGE_SIGN_MESSAGE ((psa_key_usage_t) 0x00000400) - -/** Whether the key may be used to verify a message. - * - * This flag allows the key to be used for a MAC verification operation or for - * an asymmetric message signature verification operation, if otherwise - * permitted by the key’s type and policy. - * - * For a key pair, this concerns the public key. - */ -#define PSA_KEY_USAGE_VERIFY_MESSAGE ((psa_key_usage_t) 0x00000800) - -/** Whether the key may be used to sign a message. - * - * This flag allows the key to be used for a MAC calculation operation - * or for an asymmetric signature operation, - * if otherwise permitted by the key's type and policy. - * - * For a key pair, this concerns the private key. - */ -#define PSA_KEY_USAGE_SIGN_HASH ((psa_key_usage_t) 0x00001000) - -/** Whether the key may be used to verify a message signature. - * - * This flag allows the key to be used for a MAC verification operation - * or for an asymmetric signature verification operation, - * if otherwise permitted by the key's type and policy. - * - * For a key pair, this concerns the public key. - */ -#define PSA_KEY_USAGE_VERIFY_HASH ((psa_key_usage_t) 0x00002000) - -/** Whether the key may be used to derive other keys or produce a password - * hash. - * - * This flag allows the key to be used for a key derivation operation or for - * a key agreement operation, if otherwise permitted by the key's type and - * policy. - * - * If this flag is present on all keys used in calls to - * psa_key_derivation_input_key() for a key derivation operation, then it - * permits calling psa_key_derivation_output_bytes() or - * psa_key_derivation_output_key() at the end of the operation. - */ -#define PSA_KEY_USAGE_DERIVE ((psa_key_usage_t) 0x00004000) - -/** Whether the key may be used to verify the result of a key derivation, - * including password hashing. - * - * This flag allows the key to be used: - * - * This flag allows the key to be used in a key derivation operation, if - * otherwise permitted by the key's type and policy. - * - * If this flag is present on all keys used in calls to - * psa_key_derivation_input_key() for a key derivation operation, then it - * permits calling psa_key_derivation_verify_bytes() or - * psa_key_derivation_verify_key() at the end of the operation. - */ -#define PSA_KEY_USAGE_VERIFY_DERIVATION ((psa_key_usage_t) 0x00008000) - -/**@}*/ - -/** \defgroup derivation Key derivation - * @{ - */ - -/* Key input steps are not embedded in the persistent storage, so you can - * change them if needed: it's only an ABI change. */ - -/** A secret input for key derivation. - * - * This should be a key of type #PSA_KEY_TYPE_DERIVE - * (passed to psa_key_derivation_input_key()) - * or the shared secret resulting from a key agreement - * (obtained via psa_key_derivation_key_agreement()). - * - * The secret can also be a direct input (passed to - * key_derivation_input_bytes()). In this case, the derivation operation - * may not be used to derive keys: the operation will only allow - * psa_key_derivation_output_bytes(), - * psa_key_derivation_verify_bytes(), or - * psa_key_derivation_verify_key(), but not - * psa_key_derivation_output_key(). - */ -#define PSA_KEY_DERIVATION_INPUT_SECRET ((psa_key_derivation_step_t) 0x0101) - -/** A low-entropy secret input for password hashing / key stretching. - * - * This is usually a key of type #PSA_KEY_TYPE_PASSWORD (passed to - * psa_key_derivation_input_key()) or a direct input (passed to - * psa_key_derivation_input_bytes()) that is a password or passphrase. It can - * also be high-entropy secret such as a key of type #PSA_KEY_TYPE_DERIVE or - * the shared secret resulting from a key agreement. - * - * The secret can also be a direct input (passed to - * key_derivation_input_bytes()). In this case, the derivation operation - * may not be used to derive keys: the operation will only allow - * psa_key_derivation_output_bytes(), - * psa_key_derivation_verify_bytes(), or - * psa_key_derivation_verify_key(), but not - * psa_key_derivation_output_key(). - */ -#define PSA_KEY_DERIVATION_INPUT_PASSWORD ((psa_key_derivation_step_t) 0x0102) - -/** A high-entropy additional secret input for key derivation. - * - * This is typically the shared secret resulting from a key agreement obtained - * via `psa_key_derivation_key_agreement()`. It may alternatively be a key of - * type `PSA_KEY_TYPE_DERIVE` passed to `psa_key_derivation_input_key()`, or - * a direct input passed to `psa_key_derivation_input_bytes()`. - */ -#define PSA_KEY_DERIVATION_INPUT_OTHER_SECRET \ - ((psa_key_derivation_step_t) 0x0103) - -/** A label for key derivation. - * - * This should be a direct input. - * It can also be a key of type #PSA_KEY_TYPE_RAW_DATA. - */ -#define PSA_KEY_DERIVATION_INPUT_LABEL ((psa_key_derivation_step_t) 0x0201) - -/** A salt for key derivation. - * - * This should be a direct input. - * It can also be a key of type #PSA_KEY_TYPE_RAW_DATA or - * #PSA_KEY_TYPE_PEPPER. - */ -#define PSA_KEY_DERIVATION_INPUT_SALT ((psa_key_derivation_step_t) 0x0202) - -/** An information string for key derivation. - * - * This should be a direct input. - * It can also be a key of type #PSA_KEY_TYPE_RAW_DATA. - */ -#define PSA_KEY_DERIVATION_INPUT_INFO ((psa_key_derivation_step_t) 0x0203) - -/** A seed for key derivation. - * - * This should be a direct input. - * It can also be a key of type #PSA_KEY_TYPE_RAW_DATA. - */ -#define PSA_KEY_DERIVATION_INPUT_SEED ((psa_key_derivation_step_t) 0x0204) - -/** A cost parameter for password hashing / key stretching. - * - * This must be a direct input, passed to psa_key_derivation_input_integer(). - */ -#define PSA_KEY_DERIVATION_INPUT_COST ((psa_key_derivation_step_t) 0x0205) - -/**@}*/ - -/** \defgroup helper_macros Helper macros - * @{ - */ - -/* Helper macros */ - -/** Check if two AEAD algorithm identifiers refer to the same AEAD algorithm - * regardless of the tag length they encode. - * - * \param aead_alg_1 An AEAD algorithm identifier. - * \param aead_alg_2 An AEAD algorithm identifier. - * - * \return 1 if both identifiers refer to the same AEAD algorithm, - * 0 otherwise. - * Unspecified if neither \p aead_alg_1 nor \p aead_alg_2 are - * a supported AEAD algorithm. - */ -#define MBEDTLS_PSA_ALG_AEAD_EQUAL(aead_alg_1, aead_alg_2) \ - (!(((aead_alg_1) ^ (aead_alg_2)) & \ - ~(PSA_ALG_AEAD_TAG_LENGTH_MASK | PSA_ALG_AEAD_AT_LEAST_THIS_LENGTH_FLAG))) - -/**@}*/ - -/**@}*/ - -/** \defgroup interruptible Interruptible operations - * @{ - */ - -/** Maximum value for use with \c psa_interruptible_set_max_ops() to determine - * the maximum number of ops allowed to be executed by an interruptible - * function in a single call. - */ -#define PSA_INTERRUPTIBLE_MAX_OPS_UNLIMITED UINT32_MAX - -/**@}*/ - -#endif /* PSA_CRYPTO_VALUES_H */ diff --git a/apple/CMakeLists.txt b/apple/CMakeLists.txt index 510dfa64a..85fde7328 100644 --- a/apple/CMakeLists.txt +++ b/apple/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_BUILD_TYPE Release CACHE STRING "") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) if(NOT TARGET cactus_engine) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_BINARY_DIR}/cactus) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine ${CMAKE_CURRENT_BINARY_DIR}/cactus-engine) endif() set(_DEPS cactus_engine cactus_graph cactus_kernels) diff --git a/apple/Cactus.swift b/apple/Cactus.swift deleted file mode 100644 index 0dbf015de..000000000 --- a/apple/Cactus.swift +++ /dev/null @@ -1,683 +0,0 @@ -import Foundation -import cactus - -public typealias CactusModelT = UnsafeMutableRawPointer -public typealias CactusIndexT = UnsafeMutableRawPointer -public typealias CactusStreamTranscribeT = UnsafeMutableRawPointer - -private let _frameworkInitialized: Void = { - cactus_set_telemetry_environment("swift", nil, nil) - if let bundleId = Bundle.main.bundleIdentifier { - bundleId.withCString { cactus_set_app_id($0) } - } -}() - -private func _err(_ msg: String) -> NSError { - let e = cactusGetLastError() - let desc = e.isEmpty ? msg : e - return NSError(domain: "cactus", code: -1, userInfo: [NSLocalizedDescriptionKey: desc]) -} - -private class TokenCallbackContext { - let callback: (String, UInt32) -> Void - init(callback: @escaping (String, UInt32) -> Void) { - self.callback = callback - } -} - -private func tokenCallbackBridge(token: UnsafePointer?, tokenId: UInt32, userData: UnsafeMutableRawPointer?) { - guard let token = token, let userData = userData else { return } - let context = Unmanaged.fromOpaque(userData).takeUnretainedValue() - context.callback(String(cString: token), tokenId) -} - -private let _defaultBufferSize = 65536 - -// MARK: - Telemetry - -public func cactusGetLastError() -> String { - return String(cString: cactus_get_last_error()) -} - -public func cactusSetTelemetryEnvironment(_ path: String) { - cactus_set_telemetry_environment(nil, path, nil) -} - -public func cactusSetAppId(_ appId: String) { - appId.withCString { cactus_set_app_id($0) } -} - -public func cactusTelemetryFlush() { - cactus_telemetry_flush() -} - -public func cactusTelemetryShutdown() { - cactus_telemetry_shutdown() -} - -// MARK: - Model lifecycle - -public func cactusInit(_ modelPath: String, _ corpusDir: String?, _ cacheIndex: Bool) throws -> CactusModelT { - _ = _frameworkInitialized - guard let h = cactus_init(modelPath, corpusDir, cacheIndex) else { - throw _err("Failed to initialize model") - } - return h -} - -public func cactusDestroy(_ model: CactusModelT) { - cactus_destroy(model) -} - -public func cactusReset(_ model: CactusModelT) { - cactus_reset(model) -} - -public func cactusStop(_ model: CactusModelT) { - cactus_stop(model) -} - -// MARK: - Inference - -public func cactusComplete(_ model: CactusModelT, _ messagesJson: String, _ optionsJson: String?, _ toolsJson: String?, _ onToken: ((String, UInt32) -> Void)?, _ pcmData: Data? = nil) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let callbackContext = onToken.map { TokenCallbackContext(callback: $0) } - let contextPtr = callbackContext.map { Unmanaged.passUnretained($0).toOpaque() } - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_complete( - model, - messagesJson, - bufferPtr.baseAddress, - bufferPtr.count, - optionsJson, - toolsJson, - onToken != nil ? tokenCallbackBridge : nil, - contextPtr, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_complete( - model, - messagesJson, - bufferPtr.baseAddress, - bufferPtr.count, - optionsJson, - toolsJson, - onToken != nil ? tokenCallbackBridge : nil, - contextPtr, - nil, 0 - ) - } - } - - if result < 0 { throw _err("Completion failed") } - return String(cString: buffer) -} - -public func cactusPrefill(_ model: CactusModelT, _ messagesJson: String, _ optionsJson: String?, _ toolsJson: String?, _ pcmData: Data? = nil) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_prefill( - model, - messagesJson, - bufferPtr.baseAddress, - bufferPtr.count, - optionsJson, - toolsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_prefill( - model, - messagesJson, - bufferPtr.baseAddress, - bufferPtr.count, - optionsJson, - toolsJson, - nil, 0 - ) - } - } - - if result < 0 { throw _err("Prefill failed") } - return String(cString: buffer) -} - -public func cactusTokenize(_ model: CactusModelT, _ text: String) throws -> [UInt32] { - var tokenBuffer = [UInt32](repeating: 0, count: 8192) - var tokenLen: Int = 0 - - let result = tokenBuffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_tokenize(model, text, bufferPtr.baseAddress, bufferPtr.count, &tokenLen) - } - - if result < 0 { throw _err("Tokenization failed") } - return Array(tokenBuffer.prefix(tokenLen)) -} - -public func cactusScoreWindow(_ model: CactusModelT, _ tokens: [UInt32], _ start: Int, _ end: Int, _ context: Int) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result = tokens.withUnsafeBufferPointer { tokenPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_score_window( - model, - tokenPtr.baseAddress, tokenPtr.count, - start, end, context, - bufferPtr.baseAddress, bufferPtr.count - ) - } - } - - if result < 0 { throw _err("Score window failed") } - return String(cString: buffer) -} - -public func cactusDetectLanguage(_ model: CactusModelT, _ audioPath: String?, _ optionsJson: String?, _ pcmData: Data?) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_detect_language( - model, audioPath, - bufferPtr.baseAddress, bufferPtr.count, - optionsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_detect_language(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, optionsJson, nil, 0) - } - } - - if result < 0 { throw _err("Detect language failed") } - return String(cString: buffer) -} - -public func cactusTranscribe(_ model: CactusModelT, _ audioPath: String?, _ prompt: String?, _ optionsJson: String?, _ onToken: ((String, UInt32) -> Void)?, _ pcmData: Data?) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - let callbackContext = onToken.map { TokenCallbackContext(callback: $0) } - let contextPtr = callbackContext.map { Unmanaged.passUnretained($0).toOpaque() } - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_transcribe( - model, audioPath, prompt, - bufferPtr.baseAddress, bufferPtr.count, - optionsJson, - onToken != nil ? tokenCallbackBridge : nil, contextPtr, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_transcribe( - model, audioPath, prompt, - bufferPtr.baseAddress, bufferPtr.count, - optionsJson, - onToken != nil ? tokenCallbackBridge : nil, contextPtr, - nil, 0 - ) - } - } - - if result < 0 { throw _err("Transcription failed") } - return String(cString: buffer) -} - -public func cactusStreamTranscribeStart(_ model: CactusModelT, _ optionsJson: String?) throws -> CactusStreamTranscribeT { - guard let h = cactus_stream_transcribe_start(model, optionsJson) else { - throw _err("Failed to create stream transcriber") - } - return h -} - -public func cactusStreamTranscribeProcess(_ stream: CactusStreamTranscribeT, _ pcmData: Data) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_stream_transcribe_process( - stream, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), - pcmData.count, - bufferPtr.baseAddress, - bufferPtr.count - ) - } - } - - if result < 0 { throw _err("Stream process failed") } - return String(cString: buffer) -} - -public func cactusStreamTranscribeStop(_ stream: CactusStreamTranscribeT) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_stream_transcribe_stop(stream, bufferPtr.baseAddress, bufferPtr.count) - } - - if result < 0 { throw _err("Stream stop failed") } - return String(cString: buffer) -} - -public func cactusEmbed(_ model: CactusModelT, _ text: String, _ normalize: Bool) throws -> [Float] { - var embeddingBuffer = [Float](repeating: 0, count: 4096) - var embeddingDim: Int = 0 - - let result = embeddingBuffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_embed(model, text, bufferPtr.baseAddress, bufferPtr.count, &embeddingDim, normalize) - } - - if result < 0 { throw _err("Embedding failed") } - return Array(embeddingBuffer.prefix(embeddingDim)) -} - -public func cactusImageEmbed(_ model: CactusModelT, _ imagePath: String) throws -> [Float] { - var embeddingBuffer = [Float](repeating: 0, count: 4096) - var embeddingDim: Int = 0 - - let result = embeddingBuffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_image_embed(model, imagePath, bufferPtr.baseAddress, bufferPtr.count, &embeddingDim) - } - - if result < 0 { throw _err("Image embedding failed") } - return Array(embeddingBuffer.prefix(embeddingDim)) -} - -public func cactusAudioEmbed(_ model: CactusModelT, _ audioPath: String) throws -> [Float] { - var embeddingBuffer = [Float](repeating: 0, count: 4096) - var embeddingDim: Int = 0 - - let result = embeddingBuffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_audio_embed(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, &embeddingDim) - } - - if result < 0 { throw _err("Audio embedding failed") } - return Array(embeddingBuffer.prefix(embeddingDim)) -} - -public func cactusVad(_ model: CactusModelT, _ audioPath: String?, _ optionsJson: String?, _ pcmData: Data?) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_vad( - model, audioPath, - bufferPtr.baseAddress, bufferPtr.count, - optionsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_vad(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, optionsJson, nil, 0) - } - } - - if result < 0 { throw _err("VAD failed") } - return String(cString: buffer) -} - -public func cactusDiarize(_ model: CactusModelT, _ audioPath: String?, _ optionsJson: String?, _ pcmData: Data?) throws -> String { - var buffer = [CChar](repeating: 0, count: 1 << 20) - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_diarize( - model, audioPath, - bufferPtr.baseAddress, bufferPtr.count, optionsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count - ) - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_diarize(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, optionsJson, nil, 0) - } - } - - if result < 0 { throw _err("Diarize failed") } - return String(cString: buffer) -} - -public func cactusEmbedSpeaker(_ model: CactusModelT, _ audioPath: String?, _ optionsJson: String?, _ pcmData: Data?, _ maskWeights: [Float]? = nil) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result: Int32 - if let pcmData = pcmData { - result = pcmData.withUnsafeBytes { pcmPtr in - buffer.withUnsafeMutableBufferPointer { bufferPtr in - if let maskWeights = maskWeights { - return maskWeights.withUnsafeBufferPointer { maskPtr in - cactus_embed_speaker( - model, audioPath, - bufferPtr.baseAddress, bufferPtr.count, optionsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count, - maskPtr.baseAddress, maskWeights.count - ) - } - } else { - return cactus_embed_speaker( - model, audioPath, - bufferPtr.baseAddress, bufferPtr.count, optionsJson, - pcmPtr.baseAddress?.assumingMemoryBound(to: UInt8.self), pcmData.count, - nil, 0 - ) - } - } - } - } else { - result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - if let maskWeights = maskWeights { - return maskWeights.withUnsafeBufferPointer { maskPtr in - cactus_embed_speaker(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, optionsJson, nil, 0, maskPtr.baseAddress, maskWeights.count) - } - } else { - return cactus_embed_speaker(model, audioPath, bufferPtr.baseAddress, bufferPtr.count, optionsJson, nil, 0, nil, 0) - } - } - } - - if result < 0 { throw _err("EmbedSpeaker failed") } - return String(cString: buffer) -} - -public func cactusRagQuery(_ model: CactusModelT, _ query: String, _ topK: Int) throws -> String { - var buffer = [CChar](repeating: 0, count: _defaultBufferSize) - - let result = buffer.withUnsafeMutableBufferPointer { bufferPtr in - cactus_rag_query(model, query, bufferPtr.baseAddress, bufferPtr.count, topK) - } - - if result < 0 { throw _err("RAG query failed") } - return String(cString: buffer) -} - -// MARK: - Index - -public func cactusIndexInit(_ indexDir: String, _ embeddingDim: Int) throws -> CactusIndexT { - guard let h = cactus_index_init(indexDir, embeddingDim) else { - throw _err("Failed to initialize index") - } - return h -} - -public func cactusIndexDestroy(_ index: CactusIndexT) { - cactus_index_destroy(index) -} - -public func cactusIndexAdd(_ index: CactusIndexT, _ ids: [Int32], _ documents: [String], _ embeddings: [[Float]], _ metadatas: [String]?) throws { - let count = ids.count - let embeddingDim = embeddings[0].count - - var idArray = ids - var docPtrs = documents.map { strdup($0) } - var metaPtrs: [UnsafeMutablePointer?]? = metadatas?.map { strdup($0) } - var embPtrs = embeddings.map { emb -> UnsafePointer? in - let ptr = UnsafeMutablePointer.allocate(capacity: emb.count) - ptr.initialize(from: emb, count: emb.count) - return UnsafePointer(ptr) - } - - let result = idArray.withUnsafeMutableBufferPointer { idPtr in - docPtrs.withUnsafeMutableBufferPointer { docPtr in - embPtrs.withUnsafeMutableBufferPointer { embPtr in - if let metaPtrs = metaPtrs { - var metaPtrsCopy = metaPtrs - return metaPtrsCopy.withUnsafeMutableBufferPointer { metaPtr in - cactus_index_add( - index, - idPtr.baseAddress, - unsafeBitCast(docPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - unsafeBitCast(metaPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - embPtr.baseAddress, - count, embeddingDim - ) - } - } else { - return cactus_index_add( - index, - idPtr.baseAddress, - unsafeBitCast(docPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - nil, - embPtr.baseAddress, - count, embeddingDim - ) - } - } - } - } - - docPtrs.forEach { free($0) } - metaPtrs?.forEach { free($0) } - embPtrs.forEach { if let p = $0 { UnsafeMutablePointer(mutating: p).deallocate() } } - - if result < 0 { throw _err("Failed to add documents to index") } -} - -public func cactusIndexDelete(_ index: CactusIndexT, _ ids: [Int32]) throws { - var idArray = ids - let result = idArray.withUnsafeMutableBufferPointer { idPtr in - cactus_index_delete(index, idPtr.baseAddress, ids.count) - } - - if result < 0 { throw _err("Failed to delete documents from index") } -} - -public func cactusIndexGet(_ index: CactusIndexT, _ ids: [Int32]) throws -> String { - let count = ids.count - var idArray = ids - - let docBufSize = 4096 - let embBufSize = 4096 - - let docRaw = (0.. UnsafeMutablePointer in - let p = UnsafeMutablePointer.allocate(capacity: docBufSize) - p.initialize(repeating: 0, count: docBufSize) - return p - } - let metaRaw = (0.. UnsafeMutablePointer in - let p = UnsafeMutablePointer.allocate(capacity: docBufSize) - p.initialize(repeating: 0, count: docBufSize) - return p - } - let embRaw = (0.. UnsafeMutablePointer in - let p = UnsafeMutablePointer.allocate(capacity: embBufSize) - p.initialize(repeating: 0, count: embBufSize) - return p - } - defer { - docRaw.forEach { $0.deallocate() } - metaRaw.forEach { $0.deallocate() } - embRaw.forEach { $0.deallocate() } - } - - var docBuffers: [UnsafeMutablePointer?] = docRaw.map { Optional($0) } - var docBufferSizes = [Int](repeating: docBufSize, count: count) - var metaBuffers: [UnsafeMutablePointer?] = metaRaw.map { Optional($0) } - var metaBufferSizes = [Int](repeating: docBufSize, count: count) - var embBuffers: [UnsafeMutablePointer?] = embRaw.map { Optional($0) } - var embBufferSizes = [Int](repeating: embBufSize, count: count) - - let result = idArray.withUnsafeMutableBufferPointer { idPtr in - docBuffers.withUnsafeMutableBufferPointer { docPtr in - docBufferSizes.withUnsafeMutableBufferPointer { docSzPtr in - metaBuffers.withUnsafeMutableBufferPointer { metaPtr in - metaBufferSizes.withUnsafeMutableBufferPointer { metaSzPtr in - embBuffers.withUnsafeMutableBufferPointer { embPtr in - embBufferSizes.withUnsafeMutableBufferPointer { embSzPtr in - cactus_index_get( - index, - idPtr.baseAddress, count, - unsafeBitCast(docPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - docSzPtr.baseAddress, - unsafeBitCast(metaPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - metaSzPtr.baseAddress, - unsafeBitCast(embPtr.baseAddress, to: UnsafeMutablePointer?>?.self), - embSzPtr.baseAddress - ) - } - } - } - } - } - } - } - - if result < 0 { throw _err("Failed to get from index") } - - var sb = "{\"results\":[" - for i in 0.. 0 { sb += "," } - let doc = String(cString: docRaw[i]) - let metaStr = String(cString: metaRaw[i]) - sb += "{\"document\":\"\(doc)\"" - if !metaStr.isEmpty { - sb += ",\"metadata\":\"\(metaStr)\"" - } else { - sb += ",\"metadata\":null" - } - sb += ",\"embedding\":[" - let embDim = embBufferSizes[i] - for j in 0.. 0 { sb += "," } - sb += "\(embRaw[i][j])" - } - sb += "]}" - } - sb += "]}" - return sb -} - -public func cactusIndexQuery(_ index: CactusIndexT, _ embedding: [Float], _ optionsJson: String?) throws -> String { - let resultCapacity = 1000 - var embeddingCopy = embedding - var idBuffer = [Int32](repeating: 0, count: resultCapacity) - var scoreBuffer = [Float](repeating: 0, count: resultCapacity) - var idBufferSize = resultCapacity - var scoreBufferSize = resultCapacity - - let result: Int32 - if let optStr = optionsJson { - result = embeddingCopy.withUnsafeMutableBufferPointer { embPtr in - idBuffer.withUnsafeMutableBufferPointer { idPtr in - scoreBuffer.withUnsafeMutableBufferPointer { scorePtr in - var embPtrPtr: UnsafePointer? = embPtr.baseAddress.map { UnsafePointer($0) } - var idPtrPtr: UnsafeMutablePointer? = idPtr.baseAddress - var scorePtrPtr: UnsafeMutablePointer? = scorePtr.baseAddress - var optStrCopy = optStr - return withUnsafeMutablePointer(to: &embPtrPtr) { embPtrPtrPtr in - withUnsafeMutablePointer(to: &idPtrPtr) { idPtrPtrPtr in - withUnsafeMutablePointer(to: &scorePtrPtr) { scorePtrPtrPtr in - withUnsafeMutablePointer(to: &idBufferSize) { idSizePtr in - withUnsafeMutablePointer(to: &scoreBufferSize) { scoreSizePtr in - optStrCopy.withCString { optCStr in - cactus_index_query( - index, - embPtrPtrPtr, 1, embedding.count, optCStr, - idPtrPtrPtr, idSizePtr, - scorePtrPtrPtr, scoreSizePtr - ) - } - } - } - } - } - } - } - } - } - } else { - result = embeddingCopy.withUnsafeMutableBufferPointer { embPtr in - idBuffer.withUnsafeMutableBufferPointer { idPtr in - scoreBuffer.withUnsafeMutableBufferPointer { scorePtr in - var embPtrPtr: UnsafePointer? = embPtr.baseAddress.map { UnsafePointer($0) } - var idPtrPtr: UnsafeMutablePointer? = idPtr.baseAddress - var scorePtrPtr: UnsafeMutablePointer? = scorePtr.baseAddress - return withUnsafeMutablePointer(to: &embPtrPtr) { embPtrPtrPtr in - withUnsafeMutablePointer(to: &idPtrPtr) { idPtrPtrPtr in - withUnsafeMutablePointer(to: &scorePtrPtr) { scorePtrPtrPtr in - withUnsafeMutablePointer(to: &idBufferSize) { idSizePtr in - withUnsafeMutablePointer(to: &scoreBufferSize) { scoreSizePtr in - cactus_index_query( - index, - embPtrPtrPtr, 1, embedding.count, nil, - idPtrPtrPtr, idSizePtr, - scorePtrPtrPtr, scoreSizePtr - ) - } - } - } - } - } - } - } - } - } - - if result < 0 { throw _err("Index query failed") } - - var sb = "{\"results\":[" - for i in 0.. 0 { sb += "," } - sb += "{\"id\":\(idBuffer[i]),\"score\":\(scoreBuffer[i])}" - } - sb += "]}" - return sb -} - -public func cactusIndexCompact(_ index: CactusIndexT) throws { - let result = cactus_index_compact(index) - if result < 0 { throw _err("Failed to compact index") } -} - -// MARK: - Logging - -public func cactusLogSetLevel(_ level: Int32) { - cactus_log_set_level(level) -} - -private var _logCallbackContext: ((Int32, String, String) -> Void)? - -private func logCallbackBridge(level: Int32, component: UnsafePointer?, message: UnsafePointer?, userData: UnsafeMutableRawPointer?) { - guard let component = component, let message = message else { return } - _logCallbackContext?(level, String(cString: component), String(cString: message)) -} - -public func cactusLogSetCallback(_ callback: ((Int32, String, String) -> Void)?) { - _logCallbackContext = callback - if callback != nil { - cactus_log_set_callback(logCallbackBridge, nil) - } else { - cactus_log_set_callback(nil, nil) - } -} diff --git a/apple/CmakeLists.txt b/apple/CmakeLists.txt deleted file mode 100644 index 510dfa64a..000000000 --- a/apple/CmakeLists.txt +++ /dev/null @@ -1,66 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(CactusApple LANGUAGES CXX) - -set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "") -set(CMAKE_XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "13.0" CACHE STRING "") -set(CMAKE_BUILD_TYPE Release CACHE STRING "") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) - -if(NOT TARGET cactus_engine) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus ${CMAKE_CURRENT_BINARY_DIR}/cactus) -endif() - -set(_DEPS cactus_engine cactus_graph cactus_kernels) - -if(BUILD_SHARED_LIBS) - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/stub.cpp "") - add_library(cactus_framework SHARED ${CMAKE_CURRENT_BINARY_DIR}/stub.cpp) - set_target_properties(cactus_framework PROPERTIES OUTPUT_NAME cactus) - add_dependencies(cactus_framework ${_DEPS}) - - target_link_options(cactus_framework PRIVATE - -force_load $ - -force_load $ - -force_load $ - -Wl,-dead_strip -flto - ) - - target_link_libraries(cactus_framework PUBLIC - "-framework Accelerate" "-framework CoreML" "-framework Foundation" - "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" - ) - - if(CMAKE_SYSTEM_NAME STREQUAL "iOS") - string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sysroot_lower) - if(_sysroot_lower MATCHES "simulator") - set(_CURL "${CACTUS_CURL_ROOT}/ios/simulator/libcurl.a") - else() - set(_CURL "${CACTUS_CURL_ROOT}/ios/device/libcurl.a") - endif() - else() - set(_CURL "${CACTUS_CURL_ROOT}/macos/libcurl.a") - endif() - if(EXISTS "${_CURL}") - target_link_libraries(cactus_framework PUBLIC "${_CURL}") - endif() - - set_target_properties(cactus_framework PROPERTIES - FRAMEWORK TRUE - FRAMEWORK_VERSION A - MACOSX_FRAMEWORK_IDENTIFIER com.cactuscompute.cactus - VERSION 1.0.0 - SOVERSION 1.0.0 - MACOSX_FRAMEWORK_INFO_PLIST "${CMAKE_CURRENT_SOURCE_DIR}/Info.plist" - ) - - add_custom_command(TARGET cactus_framework POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory "$/Headers" - COMMAND ${CMAKE_COMMAND} -E copy - "${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine/cactus_engine.h" - "$/Headers" - ) - - if(CMAKE_SYSTEM_NAME STREQUAL "iOS") - target_compile_definitions(cactus_framework PRIVATE TARGET_OS_IPHONE=1) - endif() -endif() diff --git a/apple/README.md b/apple/README.md index e2a5cc710..98fe929e3 100644 --- a/apple/README.md +++ b/apple/README.md @@ -1,6 +1,6 @@ # Apple Build -Builds `libcactus` for iOS and macOS. +Builds `libcactus_engine` for iOS and macOS. ## Usage @@ -16,8 +16,8 @@ bash apple/build.sh ## Output -- `libcactus-device.a` — iOS device (arm64) -- `libcactus-simulator.a` — iOS simulator (arm64) +- `libcactus_engine-device.a` — iOS device (arm64) +- `libcactus_engine-simulator.a` — iOS simulator (arm64) - `cactus-ios.xcframework/` — iOS XCFramework (device + simulator) - `cactus-macos.xcframework/` — macOS XCFramework (arm64) diff --git a/apple/build.sh b/apple/build.sh index fddffaea7..69b2ebe75 100755 --- a/apple/build.sh +++ b/apple/build.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/bin/bash +set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" @@ -32,80 +33,16 @@ echo "Vendored libcurl root: $CACTUS_CURL_ROOT" function cp_headers() { mkdir -p "$ROOT_DIR/apple/$1/$2/cactus.framework/Headers" cp "$ROOT_DIR/cactus-engine/cactus_engine.h" "$ROOT_DIR/apple/$1/$2/cactus.framework/Headers/" + mkdir -p "$ROOT_DIR/apple/$1/$2/cactus.framework/Modules" + cp "$ROOT_DIR/apple/module.modulemap" "$ROOT_DIR/apple/$1/$2/cactus.framework/Modules/" } function create_ios_xcframework_info_plist() { - cat > "$ROOT_DIR/apple/cactus-ios.xcframework/Info.plist" << 'EOF' - - - - - AvailableLibraries - - - LibraryIdentifier - ios-arm64 - LibraryPath - cactus.framework - SupportedArchitectures - - arm64 - - SupportedPlatform - ios - - - LibraryIdentifier - ios-arm64-simulator - LibraryPath - cactus.framework - SupportedArchitectures - - arm64 - - SupportedPlatform - ios - SupportedPlatformVariant - simulator - - - CFBundlePackageType - XFWK - XCFrameworkFormatVersion - 1.0 - - -EOF + cp "$ROOT_DIR/apple/cactus-ios.Info.plist" "$ROOT_DIR/apple/cactus-ios.xcframework/Info.plist" } function create_macos_xcframework_info_plist() { - cat > "$ROOT_DIR/apple/cactus-macos.xcframework/Info.plist" << 'EOF' - - - - - AvailableLibraries - - - LibraryIdentifier - macos-arm64 - LibraryPath - cactus.framework - SupportedArchitectures - - arm64 - - SupportedPlatform - macos - - - CFBundlePackageType - XFWK - XCFrameworkFormatVersion - 1.0 - - -EOF + cp "$ROOT_DIR/apple/cactus-macos.Info.plist" "$ROOT_DIR/apple/cactus-macos.xcframework/Info.plist" } function build_static_library() { @@ -133,8 +70,8 @@ function build_static_library() { cmake --build "$BUILD_DIR" --config "$CMAKE_BUILD_TYPE" -j "$n_cpu" >/dev/null mkdir -p "$APPLE_DIR" - cp "$BUILD_DIR/libcactus.a" "$APPLE_DIR/libcactus-device.a" - echo "Device static library built: $APPLE_DIR/libcactus-device.a" + cp "$BUILD_DIR/libcactus_engine.a" "$APPLE_DIR/libcactus_engine-device.a" + echo "Device static library built: $APPLE_DIR/libcactus_engine-device.a" echo "Building static library for iOS simulator..." BUILD_DIR_SIM="$APPLE_DIR/build-static-simulator" @@ -159,8 +96,8 @@ function build_static_library() { cmake --build "$BUILD_DIR_SIM" --config "$CMAKE_BUILD_TYPE" -j "$n_cpu" >/dev/null - cp "$BUILD_DIR_SIM/libcactus.a" "$APPLE_DIR/libcactus-simulator.a" - echo "Simulator static library built: $APPLE_DIR/libcactus-simulator.a" + cp "$BUILD_DIR_SIM/libcactus_engine.a" "$APPLE_DIR/libcactus_engine-simulator.a" + echo "Simulator static library built: $APPLE_DIR/libcactus_engine-simulator.a" } function build_framework() { @@ -269,8 +206,8 @@ echo "Total time: $((t1 - t0)) seconds" if [ "$BUILD_STATIC" = "true" ]; then rm -rf "$APPLE_DIR/build-static-device" "$APPLE_DIR/build-static-simulator" "$APPLE_DIR/build-static-macos" echo "Static libraries:" - echo " Device: $APPLE_DIR/libcactus-device.a" - echo " Simulator: $APPLE_DIR/libcactus-simulator.a" + echo " Device: $APPLE_DIR/libcactus_engine-device.a" + echo " Simulator: $APPLE_DIR/libcactus_engine-simulator.a" fi if [ "$BUILD_XCFRAMEWORK" = "true" ]; then diff --git a/apple/cactus-ios.Info.plist b/apple/cactus-ios.Info.plist new file mode 100644 index 000000000..55424d634 --- /dev/null +++ b/apple/cactus-ios.Info.plist @@ -0,0 +1,39 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + ios-arm64 + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + LibraryIdentifier + ios-arm64-simulator + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/apple/cactus-macos.Info.plist b/apple/cactus-macos.Info.plist new file mode 100644 index 000000000..fe06ebed8 --- /dev/null +++ b/apple/cactus-macos.Info.plist @@ -0,0 +1,25 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + macos-arm64 + LibraryPath + cactus.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + macos + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/apple/module.android.modulemap b/apple/module.android.modulemap deleted file mode 100644 index b68f641c2..000000000 --- a/apple/module.android.modulemap +++ /dev/null @@ -1,4 +0,0 @@ -module cactus { - header "cactus_ffi.h" - export * -} diff --git a/apple/module.modulemap b/apple/module.modulemap index daa014138..706a1fd96 100644 --- a/apple/module.modulemap +++ b/apple/module.modulemap @@ -1,5 +1,4 @@ framework module cactus { - umbrella header "cactus_ffi.h" + header "cactus_engine.h" export * - module * { export * } } diff --git a/assets/logo_white.png b/assets/logo_white.png old mode 100755 new mode 100644 diff --git a/bindings/README.md b/bindings/README.md index dd7d63599..e0926dcfa 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -2,7 +2,7 @@ - `python/` uses `ctypes` - `swift/` uses a C module map -- `kotlin/` uses JNI +- `kotlin/` uses JNI on Android and Kotlin/Native cinterop on iOS (KMP) - `flutter/` uses Dart FFI - `rust/` uses raw `extern "C"` declarations - `react-native/` contains a thin React Native bridge backed by the raw Kotlin and Swift bindings diff --git a/bindings/flutter/README.md b/bindings/flutter/README.md index 2cac52a61..077219048 100644 --- a/bindings/flutter/README.md +++ b/bindings/flutter/README.md @@ -4,18 +4,24 @@ Dart FFI bindings to `cactus_engine.h`. ## Integration + ```bash cactus build --apple cactus build --android ``` + -1. Copy `libcactus.so` to `jniLibs/arm64-v8a/` (Android) -2. Add `cactus-ios.xcframework` to your Xcode project (iOS) -3. Copy `cactus.dart` into your Dart source tree -4. Add `ffi` to `pubspec.yaml` + +1. Copy `android/libcactus_engine.so` to your app's `jniLibs/arm64-v8a/` +2. Add `apple/cactus-ios.xcframework` to your Xcode project (iOS) +3. Add `apple/cactus-macos.xcframework` to your Xcode project (macOS) +4. Copy `cactus.dart` into your Dart source tree +5. Add `ffi` to `pubspec.yaml` + ## Usage + ```dart import 'cactus.dart'; import 'package:ffi/ffi.dart'; @@ -24,6 +30,7 @@ final modelPath = '/path/to/model'.toNativeUtf8(); final model = cactusInit(modelPath, nullptr, false); calloc.free(modelPath); +const messagesJson = '[{"role":"user","content":"Hello"}]'; final msgs = messagesJson.toNativeUtf8(); final buf = calloc(65536); cactusComplete(model, msgs, buf.cast(), 65536, nullptr, nullptr, nullptr, nullptr, nullptr, 0); @@ -32,3 +39,4 @@ calloc.free(msgs); calloc.free(buf); cactusDestroy(model); ``` + diff --git a/bindings/flutter/cactus.dart b/bindings/flutter/cactus.dart index 11ba4f1e8..566580002 100644 --- a/bindings/flutter/cactus.dart +++ b/bindings/flutter/cactus.dart @@ -4,7 +4,7 @@ import 'dart:io'; import 'package:ffi/ffi.dart'; final DynamicLibrary _lib = (Platform.isAndroid || Platform.isLinux) - ? DynamicLibrary.open('libcactus.so') + ? DynamicLibrary.open('libcactus_engine.so') : DynamicLibrary.process(); typedef CactusModelT = Pointer; diff --git a/bindings/kotlin/Cactus.common.kt b/bindings/kotlin/Cactus.common.kt index 40f4bc6a3..42583dd85 100644 --- a/bindings/kotlin/Cactus.common.kt +++ b/bindings/kotlin/Cactus.common.kt @@ -1,13 +1,5 @@ package com.cactus -fun interface CactusTokenCallback { - fun onToken(token: String, tokenId: Int) -} - -fun interface CactusLogCallback { - fun onLog(level: Int, component: String, message: String) -} - expect fun cactusInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long expect fun cactusDestroy(handle: Long) expect fun cactusReset(handle: Long) diff --git a/bindings/kotlin/Cactus.kt b/bindings/kotlin/Cactus.kt index 28e38a773..afc4e2178 100644 --- a/bindings/kotlin/Cactus.kt +++ b/bindings/kotlin/Cactus.kt @@ -2,7 +2,7 @@ package com.cactus object CactusJNI { init { - System.loadLibrary("cactus") + System.loadLibrary("cactus_engine") } @JvmStatic external fun nativeInit(modelPath: String, corpusDir: String?, cacheIndex: Boolean): Long diff --git a/bindings/kotlin/CactusCallbacks.kt b/bindings/kotlin/CactusCallbacks.kt new file mode 100644 index 000000000..c7e4f1811 --- /dev/null +++ b/bindings/kotlin/CactusCallbacks.kt @@ -0,0 +1,9 @@ +package com.cactus + +fun interface CactusTokenCallback { + fun onToken(token: String, tokenId: Int) +} + +fun interface CactusLogCallback { + fun onLog(level: Int, component: String, message: String) +} diff --git a/bindings/kotlin/README.md b/bindings/kotlin/README.md index 10f0d5dff..a31c1cf55 100644 --- a/bindings/kotlin/README.md +++ b/bindings/kotlin/README.md @@ -2,15 +2,22 @@ JNI bridge to `cactus_engine.h` for Android, with KMP support for iOS. +The JNI bridge itself (`android/cactus_jni.cpp`, compiled into `libcactus_engine.so`) is JVM-language-agnostic. Java consumers can use it by writing an equivalent `CactusJNI.java` with `native` declarations matching the signatures in `Cactus.kt` — same library, same `Java_com_cactus_CactusJNI_*` symbols. + ## Android Integration + ```bash cactus build --android ``` + -1. Copy `android/libcactus.so` to `app/src/main/jniLibs/arm64-v8a/` -2. Copy `Cactus.kt` to your Kotlin source tree + +1. Copy `android/libcactus_engine.so` to `app/src/main/jniLibs/arm64-v8a/` +2. Copy `Cactus.kt` and `CactusCallbacks.kt` to your Kotlin source tree + + ```kotlin val handle = CactusJNI.nativeInit("/path/to/model", null, false) val buf = ByteArray(65536) @@ -18,11 +25,13 @@ CactusJNI.nativeComplete(handle, messagesJson, buf, null, null, null, null) val response = String(buf, 0, buf.indexOf(0)) CactusJNI.nativeDestroy(handle) ``` + ## Kotlin Multiplatform ``` commonMain/ Cactus.common.kt expect declarations +commonMain/ CactusCallbacks.kt shared callback interfaces androidMain/ Cactus.android.kt actual via JNI iosMain/ Cactus.ios.kt actual via cinterop ``` @@ -33,8 +42,8 @@ iosMain/ Cactus.ios.kt actual via cinterop kotlin { androidTarget() - listOf(iosArm64(), iosSimulatorArm64()).forEach { - it.compilations.getByName("main") { + listOf(iosArm64(), iosSimulatorArm64()).forEach { target -> + target.compilations.getByName("main") { cinterops { val cactus by creating { defFile("src/nativeInterop/cinterop/cactus.def") @@ -42,8 +51,9 @@ kotlin { } } } - it.binaries.framework { - linkerOpts("-L/path/to/cactus/apple", "-lcactus-device") + val libSuffix = if (target.name == "iosSimulatorArm64") "simulator" else "device" + target.binaries.framework { + linkerOpts("-L/path/to/cactus/apple", "-lcactus_engine-$libSuffix") } } } @@ -53,6 +63,6 @@ kotlin { ```kotlin val model = cactusInit("/path/to/model", null, false) -val result = cactusComplete(model, messagesJson, null, null, null) +val result = cactusComplete(model, messagesJson, null, null, null, null) cactusDestroy(model) ``` diff --git a/bindings/python/README.md b/bindings/python/README.md index 8e5f71fa6..f166b9a63 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -1,20 +1,10 @@ # Python Bindings -ctypes FFI bindings to `cactus_engine.h`. +ctypes FFI to `cactus_engine.h`. The binding module lives at +[`/python/cactus/bindings/cactus.py`](/python/cactus/bindings/cactus.py). -## Integration +It loads `libcactus_engine.{so,dylib}` from `/python/cactus/bin/`, populated by +`cactus build --python`. -```bash -cactus build --python -pip install -e python/ -``` - -## Usage - -```python -from cactus.bindings import cactus - -model = cactus.cactus_init("/path/to/model", None, False) -response = cactus.cactus_complete(model, messages_json, None, None, None) -cactus.cactus_destroy(model) -``` +For installation, the high-level API, the CLI, the server, and the transpiler, +see [`/python/README.md`](/python/). diff --git a/bindings/react-native/README.md b/bindings/react-native/README.md index fbd4561f6..b333d88b3 100644 --- a/bindings/react-native/README.md +++ b/bindings/react-native/README.md @@ -4,11 +4,14 @@ Native bridge modules over the cactus C API for iOS and Android. ## Integration + ```bash cactus build --apple cactus build --android ``` + + 1. Add the bridge files from this folder to your React Native app 2. Add the raw bindings they depend on: [`bindings/kotlin/`](/bindings/kotlin/) (Android), [`bindings/swift/`](/bindings/swift/) (Apple) 3. Register the Android package in `MainApplication.kt`: @@ -17,13 +20,16 @@ cactus build --android add(com.cactus.reactnative.CactusPackage()) } ``` + ## Usage + ```ts import Cactus from './index'; const handle = await Cactus.init('/path/to/model', null, false); -const result = await Cactus.complete(handle, messagesJson, null, null, null); +const result = await Cactus.complete(handle, messagesJson, null, null, null, false); await Cactus.destroy(handle); ``` + diff --git a/bindings/rust/README.md b/bindings/rust/README.md index 8f7029e6b..a1b3f8768 100644 --- a/bindings/rust/README.md +++ b/bindings/rust/README.md @@ -4,16 +4,18 @@ Raw `extern "C"` declarations for `cactus_engine.h`. ## Integration + ```bash cactus build ``` + -1. Copy `cactus.rs` into your project -2. Link against `libcactus.a` in your `build.rs`: +Copy [`cactus.rs`](cactus.rs) into your project (it carries its own `#[link]` +attributes) and point Cargo at the build directory: ```rust -println!("cargo:rustc-link-lib=static=cactus"); -println!("cargo:rustc-link-search=native=/path/to/cactus/cactus/build"); +// build.rs +println!("cargo:rustc-link-search=native=/path/to/cactus/cactus-engine/build"); ``` ## Usage @@ -24,11 +26,24 @@ use std::os::raw::c_char; mod cactus; -unsafe { - let path = CString::new("/path/to/model").unwrap(); - let model = cactus::cactus_init(path.as_ptr(), std::ptr::null(), false); - let mut buf = vec![0i8; 65536]; - cactus::cactus_complete(model, msgs.as_ptr(), buf.as_mut_ptr() as *mut c_char, buf.len(), std::ptr::null(), std::ptr::null(), None, std::ptr::null_mut(), std::ptr::null(), 0); - cactus::cactus_destroy(model); +fn main() { + unsafe { + let path = CString::new("/path/to/model").unwrap(); + let model = cactus::cactus_init(path.as_ptr(), std::ptr::null(), false); + + let messages = CString::new(r#"[{"role":"user","content":"Hello"}]"#).unwrap(); + let mut response = vec![0u8; 65536]; + cactus::cactus_complete( + model, + messages.as_ptr(), + response.as_mut_ptr() as *mut c_char, + response.len(), + std::ptr::null(), std::ptr::null(), + None, std::ptr::null_mut(), + std::ptr::null(), 0, + ); + + cactus::cactus_destroy(model); + } } ``` diff --git a/bindings/rust/cactus.rs b/bindings/rust/cactus.rs index e05ddeb01..d30adfa13 100644 --- a/bindings/rust/cactus.rs +++ b/bindings/rust/cactus.rs @@ -5,7 +5,15 @@ pub type CactusIndexT = *mut c_void; pub type CactusTokenCallback = Option; pub type CactusLogCallback = Option; -extern "C" { +#[cfg_attr(target_os = "macos", link(name = "Accelerate", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "CoreML", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "Foundation", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "Security", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "SystemConfiguration", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "CFNetwork", kind = "framework"))] +#[cfg_attr(target_os = "macos", link(name = "curl"))] +#[link(name = "cactus_engine", kind = "static")] +unsafe extern "C" { pub fn cactus_init(model_path: *const c_char, corpus_dir: *const c_char, cache_index: bool) -> CactusModelT; pub fn cactus_destroy(model: CactusModelT); pub fn cactus_reset(model: CactusModelT); diff --git a/bindings/swift/README.md b/bindings/swift/README.md index d84ab88d5..94d826240 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -4,16 +4,21 @@ C module map import of `cactus_engine.h`. Works on iOS and macOS. ## Integration + ```bash cactus build --apple ``` + -**XCFramework**: drag `apple/cactus-ios.xcframework` into Xcode (Embed & Sign). + +**XCFramework**: drag `apple/cactus-ios.xcframework` (or `apple/cactus-macos.xcframework`) into Xcode (Embed & Sign). The framework bundles `cactus_engine.h`, so `import cactus` works directly. -**Static library**: add `apple/libcactus-device.a` to Link Binary With Libraries, copy `module.modulemap` and `cactus_engine.h` into Header Search Paths. +**Static library**: link `apple/libcactus_engine-device.a` (or `apple/libcactus_engine-simulator.a`), copy `bindings/swift/module.modulemap` into your project, and add `cactus-engine/` to Header Search Paths so the module map can find `cactus_engine.h`. + ## Usage + ```swift import cactus @@ -23,3 +28,4 @@ cactus_complete(model, messagesJson, &buf, buf.count, nil, nil, nil, nil, nil, 0 let response = String(cString: buf) cactus_destroy(model) ``` + diff --git a/blog/gemma4.md b/blog/gemma4.md index c96f62dac..389e743e8 100644 --- a/blog/gemma4.md +++ b/blog/gemma4.md @@ -126,16 +126,16 @@ Cactus supports React Native, Flutter, Swift, Kotlin, Python, Rust, and C++. Pic | Platform | Binding | Install | |---|---|---| -| React Native | [cactus-react-native](https://github.com/cactus-compute/cactus-react-native) | `npm install cactus-react-native` | +| React Native | [React Native bindings](https://cactus.dev/react-native/) | Native bridge over the C API | | Flutter | [cactus-flutter](https://cactus.dev/flutter/) | Dart FFI bindings | | Swift (iOS/macOS) | [cactus-swift](https://cactus.dev/swift/) | XCFramework with NPU support | | Kotlin (Android) | [cactus-kotlin](https://cactus.dev/kotlin/) | JNI + Kotlin Multiplatform | -| Python | [cactus-ai](https://cactus.dev/python/) | `pip install cactus-ai` | -| Rust | [cactus-sys](https://cactus.dev/rust/) | `cargo add cactus-sys` | -| C++ | [cactus.h](https://cactus.dev/docs/cactus_engine/) | Single header, link `libcactus.a` | +| Python | [cactus-compute](https://cactus.dev/python/) | `pip install cactus-compute` | +| Rust | [Rust bindings](https://cactus.dev/rust/) | Copy `cactus.rs`, link `libcactus_engine.a` | +| C++ | [cactus.h](https://cactus.dev/docs/cactus_engine/) | Single header, link `libcactus_engine.a` | Full quickstart with code examples for every binding: [cactus.dev/docs/quickstart](https://cactus.dev/docs/quickstart/) -Pre-quantized weights for Gemma 4 and 30+ other models: [huggingface.co/cactus-compute](https://huggingface.co/cactus-compute) +Pre-quantized weights for Gemma 4 and 30+ other models: [huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute) If you're building something with on-device multimodal AI (voice agents, VR interfaces, local-first apps) we want to hear about it. Open an issue on [GitHub](https://github.com/cactus-compute/cactus), or just start building. diff --git a/blog/hybrid_transcription.md b/blog/hybrid_transcription.md index 5d43245ff..87dc15806 100644 --- a/blog/hybrid_transcription.md +++ b/blog/hybrid_transcription.md @@ -47,7 +47,7 @@ As a Mac user, you can pull down the CLI and test the latency and accuracy local ```bash brew install cactus-compute/cactus/cactus -cactus transcribe +cactus transcribe --file path/to/audio.wav ``` ## Built for the Edge, from the Ground Up @@ -77,5 +77,5 @@ Check out the open-source engine and documentation on GitHub. - [Python Binding](/python/) — Python bindings - [Swift Binding](/bindings/swift/) — Swift API - [Kotlin Binding](/bindings/kotlin/) — Kotlin API -- [Flutter Binding](/flutter/) — Flutter bindings +- [Flutter Binding](/bindings/flutter/) — Flutter bindings - [LFM2 24B Review](/blog/lfm2_24b_a2b.md) — Running large MoE models locally with Cactus \ No newline at end of file diff --git a/blog/lfm2.5_350m.md b/blog/lfm2.5_350m.md index 75ca391c7..9f2e40750 100644 --- a/blog/lfm2.5_350m.md +++ b/blog/lfm2.5_350m.md @@ -65,7 +65,7 @@ Both lock you into a vendor-specific format that may not support your model arch CPU is the universal target. Every ARM device, flagship, budget, wearable, Raspberry Pi, and most recent ones support DOTPROD or I8MM extensions. A single well-optimized CPU codepath works everywhere, the main challenge is energy-inefficiency compared to NPU. -ARM is also closing this gap. The progression from NEON to DOTPROD to I8MM to SME2 reflects a deliberate push to bring matrix-processing efficiency onto the CPU itself. Each generation narrows the energy and throughput gap with dedicated NPUs. SME2, available on Apple's M4 and upcoming ARMv9.2 cores, adds streaming matrix operations that approach NPU-class efficiency for INT8 workloads while remaining fully programmable. Cactus already compiles SME2 codepaths where available. The bet is that general-purpose CPU silicon will continue absorbing the capabilities that once justified a separate accelerator. +ARM is also closing this gap. The progression from NEON to DOTPROD to I8MM to SME2 reflects a deliberate push to bring matrix-processing efficiency onto the CPU itself. Each generation narrows the energy and throughput gap with dedicated NPUs. SME2, landing on upcoming ARMv9.2-A cores, adds streaming matrix operations that approach NPU-class efficiency for INT8 workloads while remaining fully programmable. The bet is that general-purpose CPU silicon will continue absorbing the capabilities that once justified a separate accelerator. ## Why Single Core for Decode @@ -186,7 +186,7 @@ The choice often comes down to small models that fit everywhere but are too weak Cactus exists because we took on this specific slice of the problem; I spent the last few years building foundation models and inference for tiny devices. The team's approach is systematic: profile every major ARM variant, write dedicated kernel paths for each ISA extension tier (NEON, DOTPROD, I8MM, SME2), design the model format around the worst-case memory behavior (Android's mmap limitations), and test on real production devices, not emulators. -This it not-trivial; battling with chip manufactures with different brand visions, model developers chasing benchmarks over useability and more. +This is not trivial; battling with chip manufacturers with different brand visions, model developers chasing benchmarks over usability and more. The result is that businesses shipping with Cactus don't need to worry about inference failing silently on a budget Samsung, or a model that's accurate on benchmarks but too large for half their install base. The kernel, the format, and the quantization strategy are all designed so that the same model file works correctly across the full device spectrum, with predictable RAM, predictable speed, and predictable quality. diff --git a/blog/parakeet.md b/blog/parakeet.md index 02821de6d..e62b52a37 100644 --- a/blog/parakeet.md +++ b/blog/parakeet.md @@ -158,17 +158,11 @@ cactus download nvidia/parakeet-ctc-1.1b For integrating Parakeet into your own applications, use the Python FFI bindings directly: ```python -import json from cactus import cactus_init, cactus_transcribe, cactus_destroy model = cactus_init("weights/parakeet-ctc-1.1b", None, False) -result = json.loads( - cactus_transcribe(model, "/path/to/audio.wav", None, None, None, None) -) - -if not result["success"]: - raise RuntimeError(result["error"]) +result = cactus_transcribe(model, "/path/to/audio.wav") print("\n\nFinal transcript:") print(result["response"]) @@ -179,7 +173,7 @@ cactus_destroy(model) ### 5. Use the [C API](/docs/cactus_engine.md) -The C API is the base layer all other bindings build on. Link against `libcactus` and include the FFI header: +The C API is the base layer all other bindings build on. Link against `libcactus_engine` and include the FFI header: ```c #include "cactus_engine.h" @@ -203,9 +197,9 @@ int main() { } ``` -### 6. Use the [Rust Binding](/rust/) +### 6. Use the [Rust Binding](/bindings/rust/) -Add `cactus-sys` to your `Cargo.toml` and call the FFI bindings directly: +Copy `cactus.rs` into your project (see [the README](/bindings/rust/)), link `libcactus_engine.a` from `cactus build`, and call the FFI bindings directly: ```rust use std::ffi::CString; @@ -250,20 +244,9 @@ The Swift binding exposes top-level functions that map directly to the C FFI: import Foundation let model = try cactusInit("weights/parakeet-ctc-1.1b", nil, false) - -// File-based transcription let resultJson = try cactusTranscribe(model, "/path/to/audio.wav", nil, nil, nil, nil) print(resultJson) -// Streaming transcription -let stream = try cactusStreamTranscribeStart(model, "{\"min_chunk_size\": 16000}") -// Feed PCM chunks from your audio source (16-bit, 16kHz, mono) -let partial = try cactusStreamTranscribeProcess(stream, audioChunk) -print("Partial: \(partial)") - -let finalResult = try cactusStreamTranscribeStop(stream) -print("Final: \(finalResult)") - cactusDestroy(model) ``` @@ -275,24 +258,13 @@ The Kotlin binding exposes top-level functions that map directly to the C FFI: import com.cactus.* val model = cactusInit("weights/parakeet-ctc-1.1b", null, false) - -// File-based transcription val resultJson = cactusTranscribe(model, "/path/to/audio.wav", null, null, null, null) println(resultJson) -// Streaming transcription -val stream = cactusStreamTranscribeStart(model, "{\"min_chunk_size\": 16000}") -// Feed PCM chunks from your audio source (16-bit, 16kHz, mono) -val partial = cactusStreamTranscribeProcess(stream, audioChunk) -println("Partial: $partial") - -val finalResult = cactusStreamTranscribeStop(stream) -println("Final: $finalResult") - cactusDestroy(model) ``` -### 9. Use the [Flutter Binding](/flutter/) +### 9. Use the [Flutter Binding](/bindings/flutter/) The Flutter binding brings Cactus transcription to iOS, macOS, and Android: @@ -300,19 +272,9 @@ The Flutter binding brings Cactus transcription to iOS, macOS, and Android: import 'cactus.dart'; final model = cactusInit('weights/parakeet-ctc-1.1b', null, false); - -// File-based transcription final resultJson = cactusTranscribe(model, '/path/to/audio.wav', null, null, null, null); print(resultJson); -// Streaming transcription -final stream = cactusStreamTranscribeStart(model, '{"min_chunk_size": 16000}'); -final partial = cactusStreamTranscribeProcess(stream, audioChunk); -print('Partial: $partial'); - -final finalResult = cactusStreamTranscribeStop(stream); -print('Final: $finalResult'); - cactusDestroy(model); ``` diff --git a/cactus-engine/CMakeLists.txt b/cactus-engine/CMakeLists.txt index d240d748c..1755844cf 100644 --- a/cactus-engine/CMakeLists.txt +++ b/cactus-engine/CMakeLists.txt @@ -8,41 +8,8 @@ if(NOT TARGET cactus_graph) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus-graph ${CMAKE_CURRENT_BINARY_DIR}/cactus-graph) endif() -# --- libcurl --- set(CACTUS_CURL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/libs/curl" CACHE PATH "Path to vendored libcurl") -set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") -if(APPLE) - if(CMAKE_SYSTEM_NAME STREQUAL "iOS") - string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sysroot_lower) - if(_sysroot_lower MATCHES "simulator") - set(CACTUS_CURL_LIBRARY "${CACTUS_CURL_ROOT}/ios/simulator/libcurl.a") - else() - set(CACTUS_CURL_LIBRARY "${CACTUS_CURL_ROOT}/ios/device/libcurl.a") - endif() - else() - set(CACTUS_CURL_LIBRARY "${CACTUS_CURL_ROOT}/macos/libcurl.a") - endif() - if(EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" AND EXISTS "${CACTUS_CURL_LIBRARY}") - set(CACTUS_HAS_CURL TRUE) - else() - set(CACTUS_HAS_CURL FALSE) - message(WARNING "Vendored libcurl not found at ${CACTUS_CURL_ROOT}; cloud features disabled") - endif() -elseif(ANDROID) - set(CACTUS_CURL_LIBRARY "${CACTUS_CURL_ROOT}/android/${ANDROID_ABI}/libcurl.a") - if(EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" AND EXISTS "${CACTUS_CURL_LIBRARY}") - set(CACTUS_HAS_CURL TRUE) - else() - set(CACTUS_HAS_CURL FALSE) - message(WARNING "Vendored libcurl not found for Android; cloud features disabled") - endif() -else() - find_package(CURL QUIET) - set(CACTUS_HAS_CURL ${CURL_FOUND}) -endif() - -# --- Sources --- set(ENGINE_SOURCES src/bpe.cpp src/sp.cpp @@ -53,7 +20,6 @@ set(ENGINE_SOURCES src/index.cpp src/index_ffi.cpp src/log.cpp - src/graph_ffi.cpp src/rag.cpp src/telemetry.cpp src/telemetry_impl.cpp @@ -73,52 +39,70 @@ else() endif() add_library(cactus_engine STATIC ${ENGINE_SOURCES}) - set_target_properties(cactus_engine PROPERTIES POSITION_INDEPENDENT_CODE ON + OUTPUT_NAME cactus_engine_core ) target_include_directories(cactus_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/libs - ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels/src - ${CMAKE_CURRENT_SOURCE_DIR}/../cactus-kernels/libs ) target_link_libraries(cactus_engine PUBLIC cactus_graph) -# --- curl linking --- -if(CACTUS_HAS_CURL) - if(APPLE OR ANDROID) - target_link_libraries(cactus_engine PRIVATE "${CACTUS_CURL_LIBRARY}") - target_include_directories(cactus_engine PRIVATE "${CACTUS_CURL_INCLUDE_DIR}") +if(APPLE) + target_link_libraries(cactus_engine PUBLIC + "-framework CoreML" "-framework Foundation" + "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" + ) + + if(CMAKE_SYSTEM_NAME STREQUAL "iOS") + string(TOLOWER "${CMAKE_OSX_SYSROOT}" _sysroot) + if(_sysroot MATCHES "simulator") + set(_curl "${CACTUS_CURL_ROOT}/ios/simulator/libcurl.a") + else() + set(_curl "${CACTUS_CURL_ROOT}/ios/device/libcurl.a") + endif() else() - target_link_libraries(cactus_engine PRIVATE ${CURL_LIBRARIES}) - target_include_directories(cactus_engine PRIVATE ${CURL_INCLUDE_DIRS}) + set(_curl "${CACTUS_CURL_ROOT}/macos/libcurl.a") endif() - target_compile_definitions(cactus_engine PRIVATE CACTUS_USE_CURL=1) -endif() - -# --- Apple frameworks --- -if(APPLE) - find_library(COREML_FRAMEWORK CoreML REQUIRED) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(SECURITY_FRAMEWORK Security REQUIRED) - find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) - find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) + if(EXISTS "${_curl}") + target_link_libraries(cactus_engine PRIVATE "${_curl}") + target_include_directories(cactus_engine PRIVATE "${CACTUS_CURL_ROOT}/include") + target_compile_definitions(cactus_engine PRIVATE CACTUS_USE_CURL=1) + endif() +elseif(ANDROID) + set(_mbedtls ${CMAKE_CURRENT_SOURCE_DIR}/../android/mbedtls/${ANDROID_ABI}/lib) target_link_libraries(cactus_engine PUBLIC - ${COREML_FRAMEWORK} ${FOUNDATION_FRAMEWORK} - ${SECURITY_FRAMEWORK} ${SYSTEMCONFIGURATION_FRAMEWORK} ${CFNETWORK_FRAMEWORK} + ${_mbedtls}/libmbedtls.a + ${_mbedtls}/libmbedx509.a + ${_mbedtls}/libmbedcrypto.a + log + android ) + + set(_curl "${CACTUS_CURL_ROOT}/android/${ANDROID_ABI}/libcurl.a") + if(EXISTS "${_curl}") + target_link_libraries(cactus_engine PRIVATE "${_curl}") + target_include_directories(cactus_engine PRIVATE "${CACTUS_CURL_ROOT}/include") + target_compile_definitions(cactus_engine PRIVATE CACTUS_USE_CURL=1) + endif() +else() + find_package(CURL QUIET) + if(CURL_FOUND) + target_link_libraries(cactus_engine PRIVATE ${CURL_LIBRARIES}) + target_include_directories(cactus_engine PRIVATE ${CURL_INCLUDE_DIRS}) + target_compile_definitions(cactus_engine PRIVATE CACTUS_USE_CURL=1) + endif() endif() -# --- Version --- -set(_CACTUS_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../CACTUS_VERSION") -if(EXISTS "${_CACTUS_VERSION_FILE}") - file(READ "${_CACTUS_VERSION_FILE}" _CACTUS_VERSION_CONTENT) - string(STRIP "${_CACTUS_VERSION_CONTENT}" CACTUS_VERSION_STR) - target_compile_definitions(cactus_engine PRIVATE CACTUS_COMPILE_TIME_VERSION="${CACTUS_VERSION_STR}") +set(_version_file "${CMAKE_CURRENT_SOURCE_DIR}/../CACTUS_VERSION") +if(EXISTS "${_version_file}") + file(READ "${_version_file}" _version) + string(STRIP "${_version}" _version) + target_compile_definitions(cactus_engine PRIVATE CACTUS_COMPILE_TIME_VERSION="${_version}") endif() target_compile_options(cactus_engine PRIVATE @@ -126,3 +110,59 @@ target_compile_options(cactus_engine PRIVATE $<$:-Wno-c99-extensions> $<$:-Wno-pedantic> ) + +set(_CACTUS_ENGINE_BUNDLE "${CMAKE_BINARY_DIR}/libcactus_engine.a") +set(_CACTUS_ENGINE_ARCHIVES + $ + $ + $ +) +set(_CACTUS_ENGINE_DEPS cactus_engine cactus_graph cactus_kernels) + +if(APPLE) + add_custom_target(cactus_engine_bundle ALL + COMMAND libtool -static -o ${_CACTUS_ENGINE_BUNDLE} ${_CACTUS_ENGINE_ARCHIVES} + DEPENDS ${_CACTUS_ENGINE_DEPS} + BYPRODUCTS ${_CACTUS_ENGINE_BUNDLE} + ) +else() + file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/cactus_engine_bundle.mri CONTENT + "create ${_CACTUS_ENGINE_BUNDLE}\naddlib $\naddlib $\naddlib $\nsave\nend\n" + ) + add_custom_target(cactus_engine_bundle ALL + COMMAND ${CMAKE_AR} -M < ${CMAKE_BINARY_DIR}/cactus_engine_bundle.mri + DEPENDS ${_CACTUS_ENGINE_DEPS} + BYPRODUCTS ${_CACTUS_ENGINE_BUNDLE} + ) +endif() + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + file(WRITE ${CMAKE_BINARY_DIR}/cactus_engine_shared_stub.cpp "") + add_library(cactus_engine_shared SHARED ${CMAKE_BINARY_DIR}/cactus_engine_shared_stub.cpp) + set_target_properties(cactus_engine_shared PROPERTIES OUTPUT_NAME cactus_engine) + add_dependencies(cactus_engine_shared ${_CACTUS_ENGINE_DEPS}) + + if(APPLE) + target_link_options(cactus_engine_shared PRIVATE + -Wl,-force_load,$ + -Wl,-force_load,$ + -Wl,-force_load,$ + ) + set(_curl "${CACTUS_CURL_ROOT}/macos/libcurl.a") + if(EXISTS "${_curl}") + target_link_libraries(cactus_engine_shared PRIVATE "${_curl}") + endif() + target_link_libraries(cactus_engine_shared PRIVATE + "-framework Accelerate" "-framework CoreML" "-framework Foundation" + "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" + ) + else() + target_link_options(cactus_engine_shared PRIVATE + -Wl,--whole-archive ${_CACTUS_ENGINE_ARCHIVES} -Wl,--no-whole-archive + ) + find_package(CURL QUIET) + if(CURL_FOUND) + target_link_libraries(cactus_engine_shared PRIVATE ${CURL_LIBRARIES}) + endif() + endif() +endif() diff --git a/cactus-engine/build.sh b/cactus-engine/build.sh new file mode 100755 index 000000000..68177f692 --- /dev/null +++ b/cactus-engine/build.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +cd "$(dirname "$0")" + +echo "Building cactus-engine..." + +mkdir -p build +cd build + +cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null +make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) + +echo "cactus-engine built successfully!" +echo " Static: $(pwd)/libcactus_engine.a" +echo " Shared: $(pwd)/libcactus_engine.$([ "$(uname)" = Darwin ] && echo dylib || echo so)" diff --git a/cactus-engine/cactus_engine.h b/cactus-engine/cactus_engine.h index 7f2422b2a..41af950da 100644 --- a/cactus-engine/cactus_engine.h +++ b/cactus-engine/cactus_engine.h @@ -1,6 +1,8 @@ #ifndef CACTUS_FFI_H #define CACTUS_FFI_H +#include "cactus_graph.h" + #include #include #include @@ -190,253 +192,6 @@ CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); -// cactus graph export -typedef void* cactus_graph_t; -typedef uint64_t cactus_node_t; - -typedef struct { - int32_t precision; - size_t rank; - size_t shape[8]; - size_t num_elements; - size_t byte_size; -} cactus_tensor_info_t; - -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); -CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); - -CACTUS_FFI_EXPORT int cactus_graph_input( - cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, -cactus_node_t* out_node); - -CACTUS_FFI_EXPORT int cactus_graph_set_input( - cactus_graph_t graph, cactus_node_t node, const void* data, int32_t -precision); -CACTUS_FFI_EXPORT int cactus_graph_set_external_input( - cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); -CACTUS_FFI_EXPORT int cactus_graph_mark_embedded_input( - cactus_graph_t graph, cactus_node_t node); - -CACTUS_FFI_EXPORT int cactus_graph_precision_cast( - cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, -float exponent, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_view( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_flatten( - cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_reshape( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose( - cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose_n( - cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_slice( - cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_index( - cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_concat( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, -cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_cat( - cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t -axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_matmul( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gather( - cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( - cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( - cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_bind_mmap_weights( - cactus_graph_t graph, cactus_node_t node, const char* filename); -CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( - cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_layernorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_groupnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_batchnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rms_norm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( - cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_kv_cache_state( - cactus_graph_t graph, size_t max_seq_len, size_t num_kv_heads, size_t head_dim, size_t window_size, size_t sink_size, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_kv_cache_append( - cactus_graph_t graph, cactus_node_t new_kv, cactus_node_t cache_state, size_t window_size, size_t sink_size, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention_cached( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, - cactus_node_t k_cache_state, cactus_node_t v_cache_state, - float scale, size_t position_offset, size_t window_size, size_t v_head_dim, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_conv_cache_state( - cactus_graph_t graph, size_t window_size, size_t hidden_dim, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv_cache_append( - cactus_graph_t graph, cactus_node_t new_data, cactus_node_t cache_state, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv_cache_initialize( - cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_state( - cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_write( - cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_rfft( - cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_irfft( - cactus_graph_t graph, cactus_node_t input, size_t output_length, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mel_filter_bank( - cactus_graph_t graph, size_t num_frequency_bins, size_t num_mel_filters, - float min_frequency, float max_frequency, size_t sampling_rate, - int norm_type, int scale_type, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_spectrogram( - cactus_graph_t graph, cactus_node_t waveform, cactus_node_t mel_filters, - size_t frame_length, size_t hop_length, size_t fft_length, - float power, bool center, int pad_mode, - float mel_floor, int log_mel_mode, - float dither, float preemphasis, bool remove_dc_offset, - cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_image_preprocess( - cactus_graph_t graph, cactus_node_t pixel_input, - int src_width, int src_height, int target_width, int target_height, - int patch_size, int channels, float rescale_factor, - const float* mean, const float* std_dev, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( - cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_stft( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_altup_predict( - cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_altup_correct( - cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( - cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_sample( - cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( - cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_persistent( - cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_is_populated( - cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); -CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( - cactus_graph_t graph, cactus_node_t persistent_node); - -CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, -cactus_node_t node, void** out_ptr); -CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, -cactus_node_t node, cactus_tensor_info_t* out_info); - #ifdef __cplusplus } #endif diff --git a/cactus-engine/src/init.cpp b/cactus-engine/src/init.cpp index f435f2133..14f0b353e 100644 --- a/cactus-engine/src/init.cpp +++ b/cactus-engine/src/init.cpp @@ -335,8 +335,6 @@ static bool load_corpus_index(CactusModelHandle* handle, const std::string& corp } } -std::string last_error_message; - bool matches_stop_sequence(const std::vector& generated_tokens, const std::vector>& stop_sequences) { for (const auto& stop_seq : stop_sequences) { @@ -351,10 +349,6 @@ bool matches_stop_sequence(const std::vector& generated_tokens, extern "C" { -const char* cactus_get_last_error() { - return last_error_message.c_str(); -} - cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size @@ -375,7 +369,7 @@ cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool try { auto* handle = new CactusModelHandle(); - handle->model = create_model(model_path); + handle->model = create_model(model_path_str); handle->model_name = model_name; if (!handle->model) { diff --git a/cactus-engine/test.sh b/cactus-engine/test.sh index 2769baca9..a5da20aa6 100755 --- a/cactus-engine/test.sh +++ b/cactus-engine/test.sh @@ -1,6 +1,5 @@ #!/bin/bash set -e - cd "$(dirname "$0")" PROJECT_ROOT="$(pwd)/.." @@ -8,89 +7,78 @@ ASSETS_DIR="$(pwd)/tests/assets" IOS_MODE=false ANDROID_MODE=false -ONLY_EXEC="" -MODEL_ARG="" +SUITE="" while [[ $# -gt 0 ]]; do case $1 in - --ios) - IOS_MODE=true - shift - ;; - --android) - ANDROID_MODE=true - shift - ;; - --only) - [ -z "${2:-}" ] && echo "Error: --only requires an argument" && exit 1 - ONLY_EXEC="$2" - shift 2 - ;; - --model) - [ -z "${2:-}" ] && echo "Error: --model requires an argument" && exit 1 - MODEL_ARG="$2" - shift 2 - ;; - *) - shift - ;; + --ios) IOS_MODE=true; shift ;; + --android) ANDROID_MODE=true; shift ;; + --suite) SUITE="${2:?--suite needs an argument}"; shift 2 ;; + --model) CACTUS_TEST_MODEL="${2:?--model needs an argument}"; shift 2 ;; + --transcription-model) CACTUS_TEST_TRANSCRIPTION_MODEL="${2:?--transcription-model needs an argument}"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; esac done -MODEL_DIR="${CACTUS_TEST_MODEL:-${MODEL_ARG:-$PROJECT_ROOT/weights/LFM2-VL-450M}}" +resolve_bundle_dir() { + local model="$1" label="$2" + if [ -z "$model" ]; then + echo "Error: --$label is required. Run engine tests via the CLI (cactus test ...) which sets it." >&2 + exit 2 + fi + local dir + if [[ "$model" == /* || "$model" == ./* ]]; then + dir="$model" + else + dir="$PROJECT_ROOT/weights/$(basename "$model" | tr '[:upper:]' '[:lower:]')" + fi + if [ ! -f "$dir/components/manifest.json" ]; then + echo "Bundle missing at $dir. Run: cactus transpile $model" >&2 + exit 1 + fi + echo "$dir" +} + +BUNDLE_DIR="$(resolve_bundle_dir "$CACTUS_TEST_MODEL" "model")" +TRANSCRIPTION_BUNDLE_DIR="$(resolve_bundle_dir "$CACTUS_TEST_TRANSCRIPTION_MODEL" "transcription-model")" if [ "$IOS_MODE" = true ]; then - export CACTUS_TEST_ONLY="$ONLY_EXEC" - exec "$(pwd)/tests/ios/run.sh" "$MODEL_DIR" + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + exec "$(pwd)/tests/ios/run.sh" fi - if [ "$ANDROID_MODE" = true ]; then - export CACTUS_TEST_ONLY="$ONLY_EXEC" - exec "$(pwd)/tests/android/run.sh" "$MODEL_DIR" -fi - -if [ ! -d "$MODEL_DIR" ]; then - echo "Model weights not found at $MODEL_DIR" - echo "Set CACTUS_TEST_MODEL or download weights first." - exit 1 + export CACTUS_TEST_MODEL="$BUNDLE_DIR" CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" CACTUS_TEST_SUITE="$SUITE" + exec "$(pwd)/tests/android/run.sh" fi -echo "Building and testing cactus-engine..." -echo "Model: $MODEL_DIR" - -cd "$PROJECT_ROOT/cactus" -rm -rf build -mkdir -p build -cd build -cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +echo "Model: $CACTUS_TEST_MODEL" +echo "Bundle: $BUNDLE_DIR" +echo "Transcription model: $CACTUS_TEST_TRANSCRIPTION_MODEL" +echo "Transcription bundle: $TRANSCRIPTION_BUNDLE_DIR" cd "$PROJECT_ROOT/cactus-engine/tests" -rm -rf build -mkdir -p build -cd build -cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) +rm -rf build && mkdir build && cd build +cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null +make -j"$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" -echo "" -export CACTUS_TEST_MODEL="$MODEL_DIR" +export CACTUS_TEST_MODEL="$BUNDLE_DIR" +export CACTUS_TEST_TRANSCRIPTION_MODEL="$TRANSCRIPTION_BUNDLE_DIR" export CACTUS_TEST_ASSETS="$ASSETS_DIR" export CACTUS_INDEX_PATH="$ASSETS_DIR" FAILED=0 - -if [ -n "$ONLY_EXEC" ]; then - target="./test_$ONLY_EXEC" +if [ -n "$SUITE" ]; then + target="./test_$SUITE" if [ -x "$target" ]; then - ./"$target" || FAILED=1 + "$target" || FAILED=1 else - echo "Test not found: $target" + echo "Test not found: $target" >&2 FAILED=1 fi else - for test_bin in test_*; do - [ -x "$test_bin" ] && ./"$test_bin" || FAILED=1 + for t in ./test_*; do + [ -x "$t" ] || continue + "$t" || FAILED=1 done fi - exit $FAILED diff --git a/cactus-engine/tests/CMakeLists.txt b/cactus-engine/tests/CMakeLists.txt index ebd22f639..804fa319c 100644 --- a/cactus-engine/tests/CMakeLists.txt +++ b/cactus-engine/tests/CMakeLists.txt @@ -2,76 +2,29 @@ cmake_minimum_required(VERSION 3.10) project(CactusEngineTests LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED True) - -set(CACTUS_ENGINE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..) -set(CACTUS_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) +set(CMAKE_CXX_STANDARD_REQUIRED ON) if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -pthread -Wall -Wextra -pedantic -O3") - set(CACTUS_CURL_ROOT "${CACTUS_ENGINE_ROOT}/libs/curl") - set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") - set(CACTUS_CURL_LIBRARY_MACOS "${CACTUS_CURL_ROOT}/macos/libcurl.a") - if(NOT EXISTS "${CACTUS_CURL_LIBRARY_MACOS}") - unset(CACTUS_CURL_LIBRARY_MACOS) - find_library(CACTUS_CURL_LIBRARY_MACOS NAMES curl) - endif() -else() - find_package(CURL) endif() -set(CACTUS_LIB "${CACTUS_PROJECT_ROOT}/cactus/build/libcactus.a") -if(NOT EXISTS "${CACTUS_LIB}") - message(FATAL_ERROR "Missing ${CACTUS_LIB}; build the top-level cactus target first") +if(NOT TARGET cactus_engine) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/cactus-engine) endif() -if(NOT APPLE) - find_library(MMAN_LIB mman) -else() - find_library(COREML_FRAMEWORK CoreML REQUIRED) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - find_library(SECURITY_FRAMEWORK Security REQUIRED) - find_library(SYSTEMCONFIGURATION_FRAMEWORK SystemConfiguration REQUIRED) - find_library(CFNETWORK_FRAMEWORK CFNetwork REQUIRED) -endif() - -function(cactus_link_target target_name) - target_link_libraries(${target_name} PRIVATE ${CACTUS_LIB}) - target_include_directories(${target_name} PRIVATE - ${CACTUS_ENGINE_ROOT} - ${CACTUS_PROJECT_ROOT}/cactus-graph - ${CACTUS_PROJECT_ROOT}/cactus-kernels - ${CACTUS_PROJECT_ROOT}/cactus-kernels/src - ) - if(MMAN_LIB) - target_link_libraries(${target_name} PRIVATE ${MMAN_LIB}) - endif() - if(APPLE) - target_link_libraries(${target_name} PRIVATE - ${ACCELERATE_FRAMEWORK} ${COREML_FRAMEWORK} ${FOUNDATION_FRAMEWORK} - ${SECURITY_FRAMEWORK} ${SYSTEMCONFIGURATION_FRAMEWORK} ${CFNETWORK_FRAMEWORK} - ) - if(CACTUS_CURL_LIBRARY_MACOS) - target_link_libraries(${target_name} PRIVATE ${CACTUS_CURL_LIBRARY_MACOS}) - target_include_directories(${target_name} PRIVATE ${CACTUS_CURL_INCLUDE_DIR}) - endif() - elseif(CURL_FOUND) - target_link_libraries(${target_name} PRIVATE ${CURL_LIBRARIES}) - target_include_directories(${target_name} PRIVATE ${CURL_INCLUDE_DIRS}) - endif() -endfunction() - -file(GLOB TEST_SOURCES CONFIGURE_DEPENDS "test_*.cpp") +file(GLOB TEST_SOURCES "test_*.cpp") list(REMOVE_ITEM TEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/test_utils.cpp") foreach(TEST_FILE ${TEST_SOURCES}) get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) add_executable(${TEST_NAME} ${TEST_FILE} test_utils.cpp) - cactus_link_target(${TEST_NAME}) + target_link_libraries(${TEST_NAME} PRIVATE cactus_engine) endforeach() +if(TARGET test_curl AND EXISTS "${CACTUS_CURL_ROOT}/include/curl/curl.h") + target_include_directories(test_curl PRIVATE "${CACTUS_CURL_ROOT}/include") +endif() + find_package(SDL2 QUIET CONFIG) if(NOT SDL2_FOUND) find_library(SDL2_LIBRARY NAMES SDL2 PATHS /opt/homebrew/lib /usr/local/lib) @@ -81,15 +34,15 @@ if(NOT SDL2_FOUND) endif() endif() if(SDL2_FOUND) - add_executable(chat chat.cpp test_utils.cpp) - cactus_link_target(chat) + add_executable(run run.cpp test_utils.cpp) + target_link_libraries(run PRIVATE cactus_engine) if(TARGET SDL2::SDL2) - target_link_libraries(chat PRIVATE SDL2::SDL2) + target_link_libraries(run PRIVATE SDL2::SDL2) else() - target_link_libraries(chat PRIVATE ${SDL2_LIBRARY}) - target_include_directories(chat PRIVATE ${SDL2_INCLUDE_DIR}) + target_link_libraries(run PRIVATE ${SDL2_LIBRARY}) + target_include_directories(run PRIVATE ${SDL2_INCLUDE_DIR}) endif() endif() -add_executable(asr asr.cpp) -cactus_link_target(asr) +add_executable(transcribe transcribe.cpp) +target_link_libraries(transcribe PRIVATE cactus_engine) diff --git a/cactus-engine/tests/android/CMakeLists.txt b/cactus-engine/tests/android/CMakeLists.txt index a1764f8d8..9ff1fc7c2 100644 --- a/cactus-engine/tests/android/CMakeLists.txt +++ b/cactus-engine/tests/android/CMakeLists.txt @@ -2,81 +2,22 @@ cmake_minimum_required(VERSION 3.10) project(CactusAndroidTests LANGUAGES CXX) set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED True) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wall -Wextra -pedantic -O3") +set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..) -set(CACTUS_ENGINE_DIR ${PROJECT_ROOT}/cactus-engine) -set(CACTUS_GRAPH_DIR ${PROJECT_ROOT}/cactus-graph) -set(CACTUS_KERNELS_DIR ${PROJECT_ROOT}/cactus-kernels) - -set(ANDROID_BUILD_DIR ${PROJECT_ROOT}/android/build) -set(CACTUS_LIB ${ANDROID_BUILD_DIR}/cactus/libcactus.a) - -set(CACTUS_CURL_ROOT "$ENV{CACTUS_CURL_ROOT}") -if(NOT CACTUS_CURL_ROOT) - set(CACTUS_CURL_ROOT ${CACTUS_ENGINE_DIR}/libs/curl) -endif() -set(CACTUS_CURL_INCLUDE_DIR "${CACTUS_CURL_ROOT}/include") -set(CACTUS_CURL_LIBRARY "${CACTUS_CURL_ROOT}/android/arm64-v8a/libcurl.a") - -set(CACTUS_MBEDTLS_ROOT ${PROJECT_ROOT}/android/mbedtls/arm64-v8a) -if(NOT EXISTS "${CACTUS_MBEDTLS_ROOT}") - set(CACTUS_MBEDTLS_ROOT ${PROJECT_ROOT}/libs/mbedtls/android/arm64-v8a) -endif() -set(CACTUS_MBEDTLS_CRYPTO_LIBRARY "${CACTUS_MBEDTLS_ROOT}/lib/libmbedcrypto.a") -set(CACTUS_MBEDTLS_TLS_LIBRARY "${CACTUS_MBEDTLS_ROOT}/lib/libmbedtls.a") -set(CACTUS_MBEDTLS_X509_LIBRARY "${CACTUS_MBEDTLS_ROOT}/lib/libmbedx509.a") - -if(NOT EXISTS ${CACTUS_LIB}) - message(FATAL_ERROR "Cactus library not found at: ${CACTUS_LIB}") +if(NOT TARGET cactus_engine) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../.. ${CMAKE_CURRENT_BINARY_DIR}/cactus-engine) endif() -message(STATUS "Using Cactus library: ${CACTUS_LIB}") -message(STATUS "Using vendored curl root: ${CACTUS_CURL_ROOT}") - set(TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..) file(GLOB TEST_SOURCES "${TESTS_DIR}/test_*.cpp") list(REMOVE_ITEM TEST_SOURCES "${TESTS_DIR}/test_utils.cpp") -function(link_vendored_curl_if_present target_name) - if(CACTUS_CURL_ROOT AND EXISTS "${CACTUS_CURL_INCLUDE_DIR}/curl/curl.h" AND EXISTS "${CACTUS_CURL_LIBRARY}") - if(NOT EXISTS "${CACTUS_MBEDTLS_CRYPTO_LIBRARY}" OR - NOT EXISTS "${CACTUS_MBEDTLS_TLS_LIBRARY}" OR - NOT EXISTS "${CACTUS_MBEDTLS_X509_LIBRARY}") - message(FATAL_ERROR - "Vendored Android libcurl is present but mbedTLS static libs are missing.\n" - "Expected:\n" - " ${CACTUS_MBEDTLS_CRYPTO_LIBRARY}\n" - " ${CACTUS_MBEDTLS_TLS_LIBRARY}\n" - " ${CACTUS_MBEDTLS_X509_LIBRARY}") - endif() - target_include_directories(${target_name} PRIVATE "${CACTUS_CURL_INCLUDE_DIR}") - target_link_libraries(${target_name} PRIVATE - "${CACTUS_CURL_LIBRARY}" - "${CACTUS_MBEDTLS_TLS_LIBRARY}" - "${CACTUS_MBEDTLS_X509_LIBRARY}" - "${CACTUS_MBEDTLS_CRYPTO_LIBRARY}" - ) - message(STATUS "Target ${target_name} will link vendored libcurl: ${CACTUS_CURL_LIBRARY}") - endif() -endfunction() - -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) - foreach(TEST_FILE ${TEST_SOURCES}) get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) - add_executable(${TEST_NAME} - ${TEST_FILE} - ${TESTS_DIR}/test_utils.cpp - ) - target_link_libraries(${TEST_NAME} PRIVATE ${CACTUS_LIB}) - link_vendored_curl_if_present(${TEST_NAME}) - target_include_directories(${TEST_NAME} PRIVATE - ${CACTUS_ENGINE_DIR} - ${CACTUS_GRAPH_DIR} - ${CACTUS_KERNELS_DIR} - ${CACTUS_KERNELS_DIR}/src - ) - message(STATUS "Added Android test: ${TEST_NAME}") + add_executable(${TEST_NAME} ${TEST_FILE} ${TESTS_DIR}/test_utils.cpp) + target_link_libraries(${TEST_NAME} PRIVATE cactus_engine) endforeach() + +if(TARGET test_curl AND EXISTS "${CACTUS_CURL_ROOT}/include/curl/curl.h") + target_include_directories(test_curl PRIVATE "${CACTUS_CURL_ROOT}/include") +endif() diff --git a/cactus-engine/tests/android/run.sh b/cactus-engine/tests/android/run.sh index db94a4cb2..1d2e142e6 100755 --- a/cactus-engine/tests/android/run.sh +++ b/cactus-engine/tests/android/run.sh @@ -5,12 +5,12 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" CACTUS_CURL_ROOT="${CACTUS_CURL_ROOT:-$PROJECT_ROOT/cactus-engine/libs/curl}" export CACTUS_CURL_ROOT -MODEL_NAME="$1" - -if [ -z "$MODEL_NAME" ]; then - echo "Usage: run.sh " - exit 1 -fi +for var in CACTUS_TEST_MODEL CACTUS_TEST_TRANSCRIPTION_MODEL; do + if [ -z "${!var:-}" ] || [ ! -d "${!var}" ]; then + echo "Error: $var must point to a prepared bundle (got: '${!var:-}')" >&2 + exit 1 + fi +done echo "Running Cactus tests on Android..." echo "============================" @@ -182,9 +182,6 @@ if ! adb -s "$DEVICE_ID" shell echo "test" &>/dev/null; then exit 1 fi -echo "" -echo "Step 2: Building Cactus static library for Android..." - if [ -z "$ANDROID_NDK_HOME" ]; then if [ -n "$ANDROID_HOME" ]; then ANDROID_NDK_HOME=$(ls -d "$ANDROID_HOME/ndk/"* 2>/dev/null | sort -V | tail -1) @@ -201,27 +198,15 @@ fi cmake_toolchain="$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake" android_platform=${ANDROID_PLATFORM:-android-21} -android_lib_build_dir="$PROJECT_ROOT/android/build" n_jobs=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || echo 4) -cmake -DCMAKE_TOOLCHAIN_FILE="$cmake_toolchain" \ - -DANDROID_ABI="arm64-v8a" \ - -DANDROID_PLATFORM="$android_platform" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCACTUS_CURL_ROOT="$CACTUS_CURL_ROOT" \ - -S "$PROJECT_ROOT/android" \ - -B "$android_lib_build_dir" >/dev/null - -if ! cmake --build "$android_lib_build_dir" --target cactus -j "$n_jobs" >/dev/null; then - echo "Failed to build Cactus static library" - exit 1 -fi +echo "" +echo "Step 2: Building Cactus library for Android..." -if [ ! -f "$android_lib_build_dir/cactus/libcactus.a" ]; then - echo "Static library not found at $android_lib_build_dir/cactus/libcactus.a" +if ! "$PROJECT_ROOT/android/build.sh"; then + echo "Failed to build Cactus library" exit 1 fi -echo "Static library built: $android_lib_build_dir/cactus/libcactus.a" echo "" echo "Step 3: Building Android tests..." @@ -237,6 +222,7 @@ if ! cmake -S "$android_test_dir" -B "$android_build_dir" \ -DANDROID_ABI="arm64-v8a" \ -DANDROID_PLATFORM="$android_platform" \ -DCMAKE_BUILD_TYPE=Release \ + -DCACTUS_CURL_ROOT="$CACTUS_CURL_ROOT" \ -DCMAKE_RULE_MESSAGES=OFF \ -DCMAKE_VERBOSE_MAKEFILE=OFF; then echo "Failed to configure tests" @@ -259,13 +245,13 @@ fi test_executables=() for test_exe in "${all_test_executables[@]}"; do test_name=$(basename "$test_exe" | sed 's/^test_//') - if [ -z "${CACTUS_TEST_ONLY:-}" ] || [ "$test_name" = "$CACTUS_TEST_ONLY" ]; then + if [ -z "${CACTUS_TEST_SUITE:-}" ] || [ "$test_name" = "$CACTUS_TEST_SUITE" ]; then test_executables+=("$test_exe") fi done if [ ${#test_executables[@]} -eq 0 ]; then - echo "No test executables match filter: $CACTUS_TEST_ONLY" + echo "No test executables match filter: $CACTUS_TEST_SUITE" exit 1 fi @@ -274,36 +260,53 @@ echo "Found ${#test_executables[@]} test executable(s)" echo "" echo "Step 4: Deploying to device..." -model_src="$MODEL_NAME" -model_dir=$(basename "$model_src") +model_dir=$(basename "$CACTUS_TEST_MODEL") +transcription_dir=$(basename "$CACTUS_TEST_TRANSCRIPTION_MODEL") assets_src="$PROJECT_ROOT/cactus-engine/tests/assets" device_test_dir="/data/local/tmp/cactus_tests" device_model_dir="/data/local/tmp/cactus_models" device_assets_dir="/data/local/tmp/cactus_assets" +adb -s "$DEVICE_ID" shell "rm -rf $device_test_dir $device_model_dir $device_assets_dir" adb -s "$DEVICE_ID" shell "mkdir -p $device_test_dir $device_model_dir $device_assets_dir" echo "Pushing model weights..." -adb -s "$DEVICE_ID" push "$model_src" "$device_model_dir/" +if ! adb -s "$DEVICE_ID" push "$CACTUS_TEST_MODEL" "$device_model_dir/" >/dev/null; then + echo "Failed to push model weights" + exit 1 +fi + +echo "Pushing transcription model..." +if ! adb -s "$DEVICE_ID" push "$CACTUS_TEST_TRANSCRIPTION_MODEL" "$device_model_dir/" >/dev/null; then + echo "Failed to push transcription model" + exit 1 +fi if [ -d "$assets_src" ]; then echo "Pushing test assets..." - adb -s "$DEVICE_ID" push "$assets_src" "$device_assets_dir/" + if ! adb -s "$DEVICE_ID" push "$assets_src" "$device_assets_dir/" >/dev/null; then + echo "Failed to push test assets" + exit 1 + fi fi echo "Pushing test executables..." for test_exe in "${test_executables[@]}"; do test_name=$(basename "$test_exe") - adb -s "$DEVICE_ID" push "$test_exe" "$device_test_dir/" + if ! adb -s "$DEVICE_ID" push "$test_exe" "$device_test_dir/" >/dev/null; then + echo "Failed to push $test_name" + exit 1 + fi adb -s "$DEVICE_ID" shell "chmod +x $device_test_dir/$test_name" done echo "" echo "Step 5: Running tests..." echo "------------------------" -echo "Using model path: $device_model_dir/$model_dir" -echo "Using assets path: $device_assets_dir/assets" +echo "Using model path: $device_model_dir/$model_dir" +echo "Using transcription model path: $device_model_dir/$transcription_dir" +echo "Using assets path: $device_assets_dir/assets" FAILED=0 for test_exe in "${test_executables[@]}"; do @@ -313,6 +316,7 @@ for test_exe in "${test_executables[@]}"; do if ! adb -s "$DEVICE_ID" shell "cd $device_test_dir && \ export CACTUS_TEST_MODEL=$device_model_dir/$model_dir && \ + export CACTUS_TEST_TRANSCRIPTION_MODEL=$device_model_dir/$transcription_dir && \ export CACTUS_TEST_ASSETS=$device_assets_dir/assets && \ export CACTUS_INDEX_PATH=$device_assets_dir/assets && \ export CACTUS_NO_CLOUD_TELE=${CACTUS_NO_CLOUD_TELE:-1} && \ diff --git a/cactus-engine/tests/ios/configure_xcode.rb b/cactus-engine/tests/ios/configure_xcode.rb index 0490e8e47..6a2c01c7f 100755 --- a/cactus-engine/tests/ios/configure_xcode.rb +++ b/cactus-engine/tests/ios/configure_xcode.rb @@ -21,7 +21,7 @@ def generate_app_delegate(output_path, test_files) extern_declarations = test_names.map { |name| "extern int #{name}_main();" }.join("\n") test_calls = test_names.map { |name| filter_name = name.sub(/^test_/, '') - " if (should_run(\"#{filter_name}\")) #{name}_main();" + " if (should_run(\"#{filter_name}\")) failed |= (#{name}_main() != 0);" }.join("\n") app_delegate_content = <<~OBJC @@ -29,7 +29,7 @@ def generate_app_delegate(output_path, test_files) #import "AppDelegate.h" #import #import -#include "cactus.h" +#include "cactus_engine.h" #include #{extern_declarations} @@ -73,6 +73,7 @@ def generate_app_delegate(output_path, test_files) NSString *bundlePath = [[NSBundle mainBundle] bundlePath]; [self copyFromBundle:bundlePath toDocuments:getenv("CACTUS_TEST_MODEL")]; + [self copyFromBundle:bundlePath toDocuments:getenv("CACTUS_TEST_TRANSCRIPTION_MODEL")]; [self copyFromBundle:bundlePath toDocuments:getenv("CACTUS_TEST_ASSETS")]; dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(NSEC_PER_SEC)), dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ @@ -82,9 +83,12 @@ def generate_app_delegate(output_path, test_files) return filter.empty() || filter == name; }; + int failed = 0; #{test_calls} - exit(0); + FILE* f = fopen("cactus_test.exitcode", "w"); + if (f) { fprintf(f, "%d\\n", failed); fclose(f); } + exit(failed); }); return YES; @@ -163,13 +167,12 @@ def generate_app_delegate(output_path, test_files) apple_dir = File.join(project_root, 'apple') static_lib_path = device_type == 'simulator' ? - File.join(apple_dir, 'libcactus-simulator.a') : - File.join(apple_dir, 'libcactus-device.a') + File.join(apple_dir, 'libcactus_engine-simulator.a') : + File.join(apple_dir, 'libcactus_engine-device.a') fail_with("Static library not found at: #{static_lib_path}") unless File.exist?(static_lib_path) puts "Using static library: #{static_lib_path}" -cactus_dir = File.join(project_root, 'cactus') cactus_engine_dir = File.join(project_root, 'cactus-engine') cactus_graph_dir = File.join(project_root, 'cactus-graph') cactus_kernels_dir = File.join(project_root, 'cactus-kernels') @@ -203,7 +206,7 @@ def generate_app_delegate(output_path, test_files) target.build_configurations.each do |config| config.build_settings['HEADER_SEARCH_PATHS'] ||= ['$(inherited)'] - [tests_root, cactus_dir, cactus_engine_dir, cactus_graph_dir, cactus_kernels_dir, File.join(cactus_kernels_dir, 'src')].each do |path| + [tests_root, cactus_engine_dir, cactus_graph_dir, cactus_kernels_dir, File.join(cactus_kernels_dir, 'src')].each do |path| config.build_settings['HEADER_SEARCH_PATHS'] << path unless config.build_settings['HEADER_SEARCH_PATHS'].include?(path) end if curl_root && !curl_root.empty? diff --git a/cactus-engine/tests/ios/run.sh b/cactus-engine/tests/ios/run.sh index e9ddd05f9..4125d5d1e 100755 --- a/cactus-engine/tests/ios/run.sh +++ b/cactus-engine/tests/ios/run.sh @@ -5,12 +5,12 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" CACTUS_CURL_ROOT="${CACTUS_CURL_ROOT:-$PROJECT_ROOT/cactus-engine/libs/curl}" export CACTUS_CURL_ROOT -MODEL_NAME="$1" - -if [ -z "$MODEL_NAME" ]; then - echo "Usage: run.sh " - exit 1 -fi +for var in CACTUS_TEST_MODEL CACTUS_TEST_TRANSCRIPTION_MODEL; do + if [ -z "${!var:-}" ] || [ ! -d "${!var}" ]; then + echo "Error: $var must point to a prepared bundle (got: '${!var:-}')" >&2 + exit 1 + fi +done echo "Running Cactus tests on iOS..." echo "============================" @@ -309,19 +309,21 @@ fi echo "" echo "Step 5: Bundling model weights and assets..." -model_src="$MODEL_NAME" -model_dir=$(basename "$model_src") +model_dir=$(basename "$CACTUS_TEST_MODEL") +transcription_dir=$(basename "$CACTUS_TEST_TRANSCRIPTION_MODEL") assets_src="$PROJECT_ROOT/cactus-engine/tests/assets" -if [ ! -d "$model_src" ]; then - echo "Error: model weights not found at $model_src" +echo "Copying model weights to app bundle..." +rm -rf "$app_path/$model_dir" +if ! cp -R "$CACTUS_TEST_MODEL" "$app_path/"; then + echo "Error: Could not copy model weights from $CACTUS_TEST_MODEL" exit 1 fi -echo "Copying model weights to app bundle..." -rm -rf "$app_path/$model_dir" -if ! cp -R "$model_src" "$app_path/"; then - echo "Error: Could not copy model weights from $model_src" +echo "Copying transcription model to app bundle..." +rm -rf "$app_path/$transcription_dir" +if ! cp -R "$CACTUS_TEST_TRANSCRIPTION_MODEL" "$app_path/"; then + echo "Error: Could not copy transcription model from $CACTUS_TEST_TRANSCRIPTION_MODEL" exit 1 fi @@ -349,18 +351,30 @@ if [ "$device_type" = "simulator" ]; then fi echo "Launching tests..." - echo "Using model path: $model_dir" - echo "Using assets path: assets" + echo "Using model path: $model_dir" + echo "Using transcription model path: $transcription_dir" + echo "Using assets path: assets" sim_env=( "SIMCTL_CHILD_CACTUS_TEST_MODEL=$model_dir" + "SIMCTL_CHILD_CACTUS_TEST_TRANSCRIPTION_MODEL=$transcription_dir" "SIMCTL_CHILD_CACTUS_TEST_ASSETS=assets" "SIMCTL_CHILD_CACTUS_INDEX_PATH=assets" "SIMCTL_CHILD_CACTUS_NO_CLOUD_TELE=${CACTUS_NO_CLOUD_TELE:-1}" - "SIMCTL_CHILD_CACTUS_TEST_ONLY=${CACTUS_TEST_ONLY:-}" + "SIMCTL_CHILD_CACTUS_TEST_ONLY=${CACTUS_TEST_SUITE:-}" ) - env "${sim_env[@]}" xcrun simctl launch --console-pty "$device_uuid" "$bundle_id" + env "${sim_env[@]}" xcrun simctl launch --console-pty --terminate-running-process "$device_uuid" "$bundle_id" + + data_container=$(xcrun simctl get_app_container "$device_uuid" "$bundle_id" data 2>/dev/null || true) + exitcode_file="$data_container/Documents/cactus_test.exitcode" + if [ -n "$data_container" ] && [ -f "$exitcode_file" ]; then + SUITE_EXIT=$(tr -d '[:space:]' < "$exitcode_file") + [[ "$SUITE_EXIT" =~ ^[0-9]+$ ]] || SUITE_EXIT=1 + else + echo "Could not retrieve exit-code marker from simulator" + SUITE_EXIT=1 + fi else echo "Installing on: $device_name" @@ -375,19 +389,22 @@ else echo "Launching tests..." echo "(Logs will be fetched from device after completion)" - echo "Using model path: $model_dir" - echo "Using assets path: assets" + echo "Using model path: $model_dir" + echo "Using transcription model path: $transcription_dir" + echo "Using assets path: assets" device_env=( "DEVICECTL_CHILD_CACTUS_TEST_MODEL=$model_dir" + "DEVICECTL_CHILD_CACTUS_TEST_TRANSCRIPTION_MODEL=$transcription_dir" "DEVICECTL_CHILD_CACTUS_TEST_ASSETS=assets" "DEVICECTL_CHILD_CACTUS_INDEX_PATH=assets" "DEVICECTL_CHILD_CACTUS_NO_CLOUD_TELE=${CACTUS_NO_CLOUD_TELE:-1}" - "DEVICECTL_CHILD_CACTUS_TEST_ONLY=${CACTUS_TEST_ONLY:-}" + "DEVICECTL_CHILD_CACTUS_TEST_ONLY=${CACTUS_TEST_SUITE:-}" ) env "${device_env[@]}" \ - xcrun devicectl device process launch --device "$device_uuid" "$bundle_id" 2>&1 || true + xcrun devicectl device process launch --device "$device_uuid" "$bundle_id" 2>&1 + SUITE_EXIT=$? echo "Waiting for tests to complete..." max_wait=300 @@ -415,6 +432,11 @@ else --domain-type appDataContainer \ --source Documents/cactus_test.log \ --destination "$log_dir/cactus_test.log" 2>/dev/null || true + xcrun devicectl device copy from --device "$device_uuid" \ + --domain-identifier "$bundle_id" \ + --domain-type appDataContainer \ + --source Documents/cactus_test.exitcode \ + --destination "$log_dir/exitcode" 2>/dev/null || true if [ -f "$log_dir/cactus_test.log" ]; then echo "=== Device Test Output ===" @@ -423,8 +445,21 @@ else else echo "Could not retrieve test logs from device" fi + + if [ -f "$log_dir/exitcode" ]; then + SUITE_EXIT=$(tr -d '[:space:]' < "$log_dir/exitcode") + [[ "$SUITE_EXIT" =~ ^[0-9]+$ ]] || SUITE_EXIT=1 + else + echo "Could not retrieve exit-code marker from device" + SUITE_EXIT=1 + fi rm -rf "$log_dir" fi echo "" -echo "Tests complete." +if [ "${SUITE_EXIT:-1}" -eq 0 ]; then + echo "All tests passed." +else + echo "Some tests failed." +fi +exit "${SUITE_EXIT:-1}" diff --git a/cactus-engine/tests/chat.cpp b/cactus-engine/tests/run.cpp similarity index 99% rename from cactus-engine/tests/chat.cpp rename to cactus-engine/tests/run.cpp index b578d5b16..7fcdcb0fb 100644 --- a/cactus-engine/tests/chat.cpp +++ b/cactus-engine/tests/run.cpp @@ -585,7 +585,7 @@ int main(int argc, char** argv) { } input = record_prompt.empty() ? "Transcribe or respond to this audio." : record_prompt; #else - std::cerr << "Recording requires SDL2, but this chat binary was built without SDL2.\n"; + std::cerr << "Recording requires SDL2, but this binary was built without SDL2.\n"; continue; #endif } diff --git a/tests/test_curl.cpp b/cactus-engine/tests/test_curl.cpp similarity index 58% rename from tests/test_curl.cpp rename to cactus-engine/tests/test_curl.cpp index 222216204..e73296c1c 100644 --- a/tests/test_curl.cpp +++ b/cactus-engine/tests/test_curl.cpp @@ -6,40 +6,24 @@ #include #include -#if __has_include() #include -#define CACTUS_TEST_HAS_CURL 1 -#else -#define CACTUS_TEST_HAS_CURL 0 -#endif bool test_curl_version_info() { -#if !CACTUS_TEST_HAS_CURL - return false; -#else curl_version_info_data* info = curl_version_info(CURLVERSION_NOW); if (!info) return false; if (!info->version || std::string(info->version).empty()) return false; if (!info->host || std::string(info->host).empty()) return false; return true; -#endif } bool test_curl_easy_init() { -#if !CACTUS_TEST_HAS_CURL - return false; -#else CURL* handle = curl_easy_init(); if (!handle) return false; curl_easy_cleanup(handle); return true; -#endif } bool test_curl_url_api() { -#if !CACTUS_TEST_HAS_CURL - return false; -#else CURL* handle = curl_easy_init(); if (!handle) return false; @@ -51,39 +35,10 @@ bool test_curl_url_api() { curl_easy_cleanup(handle); return ok; -#endif -} - -bool test_curl_http_request() { -#if !CACTUS_TEST_HAS_CURL - return false; -#else - CURL* handle = curl_easy_init(); - if (!handle) return false; - - bool ok = true; - ok = ok && (curl_easy_setopt(handle, CURLOPT_URL, "https://example.com/") == CURLE_OK); - ok = ok && (curl_easy_setopt(handle, CURLOPT_NOBODY, 1L) == CURLE_OK); - ok = ok && (curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L) == CURLE_OK); - ok = ok && (curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 3000L) == CURLE_OK); - ok = ok && (curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L) == CURLE_OK); - if (!ok) { - curl_easy_cleanup(handle); - return false; - } - - CURLcode rc = curl_easy_perform(handle); - long http_code = 0; - curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_code); - curl_easy_cleanup(handle); - - return (rc == CURLE_OK) && (http_code >= 100); -#endif } struct CurlHttpCheck { bool pass = false; - bool skip = false; std::string reason; long http_code = 0; std::string body_preview; @@ -96,22 +51,7 @@ static size_t curl_write_to_string(char* ptr, size_t size, size_t nmemb, void* u return size * nmemb; } -static bool curl_supports_protocol(const char* name) { -#if !CACTUS_TEST_HAS_CURL - (void)name; - return false; -#else - curl_version_info_data* info = curl_version_info(CURLVERSION_NOW); - if (!info || !info->protocols) return false; - for (const char* const* p = info->protocols; *p; ++p) { - if (std::string(*p) == name) return true; - } - return false; -#endif -} - static void configure_curl_tls(CURL* handle) { -#if CACTUS_TEST_HAS_CURL if (!handle) return; curl_easy_setopt(handle, CURLOPT_SSL_VERIFYPEER, 1L); curl_easy_setopt(handle, CURLOPT_SSL_VERIFYHOST, 2L); @@ -127,21 +67,11 @@ static void configure_curl_tls(CURL* handle) { curl_easy_setopt(handle, CURLOPT_CAPATH, "/system/etc/security/cacerts"); } #endif -#else - (void)handle; -#endif } CurlHttpCheck run_curl_http_request_check() { -#if !CACTUS_TEST_HAS_CURL - return {false, true, "curl/curl.h not available in include path"}; -#else - if (!curl_supports_protocol("https")) { - return {false, false, "libcurl built without HTTPS protocol support", 0, ""}; - } - CURL* handle = curl_easy_init(); - if (!handle) return {false, false, "curl_easy_init returned null", 0, ""}; + if (!handle) return {false, "curl_easy_init returned null", 0, ""}; bool opts_ok = true; std::string body; @@ -156,7 +86,7 @@ CurlHttpCheck run_curl_http_request_check() { configure_curl_tls(handle); if (!opts_ok) { curl_easy_cleanup(handle); - return {false, false, "failed to configure curl options", 0, ""}; + return {false, "failed to configure curl options", 0, ""}; } CURLcode rc = curl_easy_perform(handle); @@ -167,28 +97,21 @@ CurlHttpCheck run_curl_http_request_check() { if (rc == CURLE_OK && http_code >= 200 && http_code < 400) { if (body.find("badssl.com") != std::string::npos) { - return {true, false, "", http_code, body_preview}; + return {true, "", http_code, body_preview}; } - std::ostringstream oss; - oss << "https request succeeded but response body did not contain expected marker"; - return {false, false, oss.str(), http_code, body_preview}; + return {false, "https request succeeded but response body did not contain expected marker", + http_code, body_preview}; } std::ostringstream oss; oss << "request failed rc=" << static_cast(rc) << " (" << curl_easy_strerror(rc) << "), http_code=" << http_code; - return {false, false, oss.str(), http_code, body_preview}; -#endif + return {false, oss.str(), http_code, body_preview}; } int main() { TestUtils::TestRunner runner("Curl Tests"); -#if !CACTUS_TEST_HAS_CURL - runner.log_skip("curl_headers", "curl/curl.h not available in include path"); - runner.print_summary(); - return 0; -#else CURLcode init_rc = curl_global_init(CURL_GLOBAL_DEFAULT); if (init_rc != CURLE_OK) { runner.run_test("global_init", false); @@ -200,23 +123,18 @@ int main() { runner.run_test("easy_init", test_curl_easy_init()); runner.run_test("url_api", test_curl_url_api()); CurlHttpCheck http_check = run_curl_http_request_check(); - if (http_check.skip) { - runner.log_skip("http_request", http_check.reason); - } else { - runner.run_test("http_request", http_check.pass); - if (http_check.http_code > 0) { - std::cout << " http_code: " << http_check.http_code << "\n"; - } - if (!http_check.body_preview.empty()) { - std::cout << " body_preview: " << http_check.body_preview << "\n"; - } - if (!http_check.pass && !http_check.reason.empty()) { - std::cout << " reason: " << http_check.reason << "\n"; - } + runner.run_test("http_request", http_check.pass); + if (http_check.http_code > 0) { + std::cout << " http_code: " << http_check.http_code << "\n"; + } + if (!http_check.body_preview.empty()) { + std::cout << " body_preview: " << http_check.body_preview << "\n"; + } + if (!http_check.pass && !http_check.reason.empty()) { + std::cout << " reason: " << http_check.reason << "\n"; } curl_global_cleanup(); runner.print_summary(); return runner.all_passed() ? 0 : 1; -#endif } diff --git a/cactus-engine/tests/test_embed.cpp b/cactus-engine/tests/test_embed.cpp index 6c19e6e0d..41c913659 100644 --- a/cactus-engine/tests/test_embed.cpp +++ b/cactus-engine/tests/test_embed.cpp @@ -5,7 +5,7 @@ using namespace EngineTestUtils; static const char* g_model_path = std::getenv("CACTUS_TEST_MODEL"); -static const char* g_transcribe_model_path = std::getenv("CACTUS_TEST_TRANSCRIBE_MODEL"); +static const char* g_transcription_model_path = std::getenv("CACTUS_TEST_TRANSCRIPTION_MODEL"); static const char* g_assets_path = std::getenv("CACTUS_TEST_ASSETS"); bool test_embeddings() { @@ -18,16 +18,21 @@ bool test_embeddings() { const char* texts[] = {"My name is Henry Ndubuaku", "Your name is Henry Ndubuaku"}; std::vector emb1(2048), emb2(2048); - size_t dim1, dim2; + size_t dim1 = 0, dim2 = 0; Timer t1; - cactus_embed(model, texts[0], emb1.data(), emb1.size() * sizeof(float), &dim1, true); + int rc1 = cactus_embed(model, texts[0], emb1.data(), emb1.size() * sizeof(float), &dim1, true); double time1 = t1.elapsed_ms(); Timer t2; - cactus_embed(model, texts[1], emb2.data(), emb2.size() * sizeof(float), &dim2, true); + int rc2 = cactus_embed(model, texts[1], emb2.data(), emb2.size() * sizeof(float), &dim2, true); double time2 = t2.elapsed_ms(); + if (rc1 < 0 || rc2 < 0 || dim1 == 0 || dim1 != dim2 || dim1 > emb1.size()) { + cactus_destroy(model); + return false; + } + float similarity = 0; for (size_t i = 0; i < dim1; ++i) { similarity += emb1[i] * emb2[i]; @@ -48,21 +53,13 @@ static bool test_image_embeddings() { << "║ IMAGE EMBEDDING TEST ║\n" << "╚══════════════════════════════════════════╝\n"; - if (!g_model_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_MODEL not set\n"; - return true; - } - std::string image_path = std::string(g_assets_path) + "/test_monkey.png"; const size_t buffer_size = 1024 * 1024 * 4; std::vector embeddings(buffer_size / sizeof(float)); size_t embedding_dim = 0; cactus_model_t model = cactus_init(g_model_path, nullptr, false); - if (!model) { - std::cout << "⊘ SKIP │ Model doesn't support image embeddings\n"; - return true; - } + if (!model) return false; Timer t; int result = cactus_image_embed(model, image_path.c_str(), embeddings.data(), buffer_size, &embedding_dim); @@ -70,11 +67,6 @@ static bool test_image_embeddings() { cactus_destroy(model); - if (result == -1) { - std::cout << "⊘ SKIP │ Model doesn't support image embeddings\n"; - return true; - } - std::cout << "├─ Embedding dim: " << embedding_dim << "\n" << "└─ Time: " << std::fixed << std::setprecision(2) << elapsed << "ms" << std::endl; @@ -86,20 +78,12 @@ static bool test_audio_embeddings() { << "║ AUDIO EMBEDDING TEST ║\n" << "╚══════════════════════════════════════════╝\n"; - if (!g_transcribe_model_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_TRANSCRIBE_MODEL not set\n"; - return true; - } - const size_t buffer_size = 1024 * 1024; std::vector embeddings(buffer_size / sizeof(float)); size_t embedding_dim = 0; - cactus_model_t model = cactus_init(g_transcribe_model_path, nullptr, false); - if (!model) { - std::cout << "⊘ SKIP │ Failed to init audio model\n"; - return true; - } + cactus_model_t model = cactus_init(g_transcription_model_path, nullptr, false); + if (!model) return false; std::string audio_path = std::string(g_assets_path) + "/test.wav"; Timer t; @@ -108,11 +92,6 @@ static bool test_audio_embeddings() { cactus_destroy(model); - if (result == -1) { - std::cout << "⊘ SKIP │ Model doesn't support audio embeddings\n"; - return true; - } - std::cout << "├─ Embedding dim: " << embedding_dim << "\n" << "└─ Time: " << std::fixed << std::setprecision(2) << elapsed << "ms" << std::endl; diff --git a/cactus-engine/tests/test_llm.cpp b/cactus-engine/tests/test_llm.cpp index ebbb097ac..56382474d 100644 --- a/cactus-engine/tests/test_llm.cpp +++ b/cactus-engine/tests/test_llm.cpp @@ -561,19 +561,15 @@ bool test_1k_context() { int main() { TestUtils::TestRunner runner("LLM Tests"); - const char* only = std::getenv("CACTUS_TEST_ONLY"); - auto should_run = [&](const char* name) { - return only == nullptr || std::string(only) == name; - }; - if (should_run("1k_context")) runner.run_test("1k_context", test_1k_context()); - if (should_run("streaming")) runner.run_test("streaming", test_streaming()); - if (should_run("prefill")) runner.run_test("prefill", test_prefill()); - if (should_run("prefill_idempotent_reuse")) runner.run_test("prefill_idempotent_reuse", test_prefill_idempotent_reuse()); - if (should_run("prefill_prefix_extension_reuse")) runner.run_test("prefill_prefix_extension_reuse", test_prefill_prefix_extension_reuse()); - if (should_run("prefill_invalidated_on_message_change")) runner.run_test("prefill_invalidated_on_message_change", test_prefill_invalidated_on_message_change()); - if (should_run("tool_calls")) runner.run_test("tool_calls", test_tool_call()); - if (should_run("tool_multiple_tool_call_invocations")) runner.run_test("tool_multiple_tool_call_invocations", test_multiple_tool_call_invocations()); - if (should_run("tool_calls_with_three_tools")) runner.run_test("tool_calls_with_three_tools", test_tool_call_with_three_tools()); + runner.run_test("1k_context", test_1k_context()); + runner.run_test("streaming", test_streaming()); + runner.run_test("prefill", test_prefill()); + runner.run_test("prefill_idempotent_reuse", test_prefill_idempotent_reuse()); + runner.run_test("prefill_prefix_extension_reuse", test_prefill_prefix_extension_reuse()); + runner.run_test("prefill_invalidated_on_message_change", test_prefill_invalidated_on_message_change()); + runner.run_test("tool_calls", test_tool_call()); + runner.run_test("tool_multiple_tool_call_invocations", test_multiple_tool_call_invocations()); + runner.run_test("tool_calls_with_three_tools", test_tool_call_with_three_tools()); runner.print_summary(); return runner.all_passed() ? 0 : 1; } diff --git a/cactus-engine/tests/test_rag.cpp b/cactus-engine/tests/test_rag.cpp index 5aad3b6fc..14d2c0398 100644 --- a/cactus-engine/tests/test_rag.cpp +++ b/cactus-engine/tests/test_rag.cpp @@ -1,7 +1,6 @@ #include "test_utils.h" #include #include -#include #include #include @@ -37,44 +36,8 @@ bool test_rag() { << "║ RAG TEST ║\n" << "╚══════════════════════════════════════════╝\n"; - if (!g_model_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_MODEL not set\n"; - return true; - } - - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - std::string corpus_dir = std::string(g_assets_path) + "/rag_corpus"; - DIR* dir = opendir(corpus_dir.c_str()); - if (!dir) { - std::cout << "⊘ SKIP │ RAG corpus directory not found at " << corpus_dir << "\n"; - return true; - } - - bool has_corpus_files = false; - struct dirent* entry; - while ((entry = readdir(dir)) != nullptr) { - std::string name = entry->d_name; - if (name.size() > 4 && name.substr(name.size() - 4) == ".txt") { - has_corpus_files = true; - break; - } - if (name.size() > 3 && name.substr(name.size() - 3) == ".md") { - has_corpus_files = true; - break; - } - } - closedir(dir); - - if (!has_corpus_files) { - std::cout << "⊘ SKIP │ No .txt or .md files found in " << corpus_dir << "\n"; - return true; - } - std::cout << "├─ Corpus dir: " << corpus_dir << "\n"; std::cout << "├─ Initializing model with RAG...\n"; diff --git a/cactus-engine/tests/test_stt.cpp b/cactus-engine/tests/test_stt.cpp index 55f9b2ca0..56b9fdd73 100644 --- a/cactus-engine/tests/test_stt.cpp +++ b/cactus-engine/tests/test_stt.cpp @@ -6,7 +6,7 @@ using namespace EngineTestUtils; -static const char* g_model_path = std::getenv("CACTUS_TEST_MODEL"); +static const char* g_transcription_model_path = std::getenv("CACTUS_TEST_TRANSCRIPTION_MODEL"); static const char* g_assets_path = std::getenv("CACTUS_TEST_ASSETS"); bool test_transcription() { @@ -14,16 +14,7 @@ bool test_transcription() { << "║ TRANSCRIPTION TEST ║\n" << "╚══════════════════════════════════════════╝\n"; - if (!g_model_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_MODEL not set\n"; - return true; - } - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - - cactus_model_t model = cactus_init(g_model_path, nullptr, false); + cactus_model_t model = cactus_init(g_transcription_model_path, nullptr, false); if (!model) { std::cerr << "[✗] Failed to initialize model\n"; return false; @@ -67,16 +58,7 @@ bool test_transcription_pcm() { << "║ TRANSCRIPTION PCM TEST ║\n" << "╚══════════════════════════════════════════╝\n"; - if (!g_model_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_MODEL not set\n"; - return true; - } - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - - cactus_model_t model = cactus_init(g_model_path, nullptr, false); + cactus_model_t model = cactus_init(g_transcription_model_path, nullptr, false); if (!model) { std::cerr << "[✗] Failed to initialize model\n"; return false; diff --git a/cactus-engine/tests/test_telemetry.cpp b/cactus-engine/tests/test_telemetry.cpp index 42f18414e..632f55779 100644 --- a/cactus-engine/tests/test_telemetry.cpp +++ b/cactus-engine/tests/test_telemetry.cpp @@ -48,12 +48,6 @@ std::string random_project_id() { return std::string(buf); } -enum class CloudTelemetryTestResult { - Passed, - Failed, - Skipped, -}; - bool test_record_many_then_flush() { const std::string cache_dir = make_temp_dir("cactus_record_many_flush"); @@ -156,6 +150,12 @@ bool test_record_and_flush_race_no_deadlock() { return all_completed && event_count == expected_event_count; } +enum class CloudTelemetryTestResult { + Passed, + Failed, + Skipped, +}; + CloudTelemetryTestResult test_cloud_upload_record_then_flush() { const char* no_cloud_tele = std::getenv("CACTUS_NO_CLOUD_TELE"); if (no_cloud_tele && no_cloud_tele[0] != '\0') { diff --git a/cactus-engine/tests/test_vlm.cpp b/cactus-engine/tests/test_vlm.cpp index 77a49717b..e8aeea925 100644 --- a/cactus-engine/tests/test_vlm.cpp +++ b/cactus-engine/tests/test_vlm.cpp @@ -1,5 +1,4 @@ #include "test_utils.h" -#include #include #include @@ -15,16 +14,6 @@ static const char* g_options = R"({ })"; bool test_vlm_multiturn() { - std::string model_path_str(g_model_path ? g_model_path : ""); - - std::string vision_file = model_path_str + "/vision_patch_embedder_input_proj.weights"; - std::ifstream vf(vision_file); - if (!vf.good()) { - std::cout << "Skipping VLM multi-turn test: vision weights not found." << std::endl; - return true; - } - vf.close(); - std::cout << "\n╔══════════════════════════════════════════╗\n" << "║ VLM MULTI-TURN TEST ║\n" << "╚══════════════════════════════════════════╝\n"; @@ -107,35 +96,8 @@ bool test_prefill_invalidated_on_message_change_vlm() { << "║" << std::setw(42) << std::left << " PREFILL INVALIDATION (VLM) TEST" << "║\n" << "╚══════════════════════════════════════════╝\n"; - std::string model_path_str(g_model_path ? g_model_path : ""); - std::string vision_file = model_path_str + "/vision_patch_embedder_input_proj.weights"; - std::ifstream vf(vision_file); - if (!vf.good()) { - std::cout << "⊘ SKIP │ Vision weights not found\n"; - return true; - } - vf.close(); - - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - std::string prefill_img_path = std::string(g_assets_path) + "/test_monkey.png"; - std::ifstream imgf(prefill_img_path); - if (!imgf.good()) { - std::cout << "⊘ SKIP │ test_monkey.png not found\n"; - return true; - } - imgf.close(); - std::string complete_img_path = std::string(g_assets_path) + "/test_thing.png"; - std::ifstream imgf2(complete_img_path); - if (!imgf2.good()) { - std::cout << "⊘ SKIP │ test_thing.png not found\n"; - return true; - } - imgf2.close(); cactus_model_t model = cactus_init(g_model_path, nullptr, false); if (!model) { @@ -197,35 +159,8 @@ bool test_prefill_with_images() { << "║" << std::setw(42) << std::left << " PREFILL WITH IMAGES TEST" << "║\n" << "╚══════════════════════════════════════════╝\n"; - std::string model_path_str(g_model_path ? g_model_path : ""); - std::string vision_file = model_path_str + "/vision_patch_embedder_input_proj.weights"; - std::ifstream vf(vision_file); - if (!vf.good()) { - std::cout << "⊘ SKIP │ Vision weights not found\n"; - return true; - } - vf.close(); - - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - std::string prefill_img_path = std::string(g_assets_path) + "/test_monkey.png"; - std::ifstream imgf(prefill_img_path); - if (!imgf.good()) { - std::cout << "⊘ SKIP │ test_monkey.png not found\n"; - return true; - } - imgf.close(); - std::string extension_img_path = std::string(g_assets_path) + "/test_thing.png"; - std::ifstream imgf2(extension_img_path); - if (!imgf2.good()) { - std::cout << "⊘ SKIP │ test_thing.png not found\n"; - return true; - } - imgf2.close(); cactus_model_t model = cactus_init(g_model_path, nullptr, false); if (!model) { @@ -302,35 +237,8 @@ bool test_prefill_prefix_extension_reuse_vlm() { << "║" << std::setw(42) << std::left << " PREFILL PREFIX EXTENSION (VLM)" << "║\n" << "╚══════════════════════════════════════════╝\n"; - std::string model_path_str(g_model_path ? g_model_path : ""); - std::string vision_file = model_path_str + "/vision_patch_embedder_input_proj.weights"; - std::ifstream vf(vision_file); - if (!vf.good()) { - std::cout << "⊘ SKIP │ Vision weights not found\n"; - return true; - } - vf.close(); - - if (!g_assets_path) { - std::cout << "⊘ SKIP │ CACTUS_TEST_ASSETS not set\n"; - return true; - } - std::string base_img_path = std::string(g_assets_path) + "/test_monkey.png"; - std::ifstream imgf(base_img_path); - if (!imgf.good()) { - std::cout << "⊘ SKIP │ test_monkey.png not found\n"; - return true; - } - imgf.close(); - std::string extension_img_path = std::string(g_assets_path) + "/test_thing.png"; - std::ifstream imgf2(extension_img_path); - if (!imgf2.good()) { - std::cout << "⊘ SKIP │ test_thing.png not found\n"; - return true; - } - imgf2.close(); cactus_model_t model = cactus_init(g_model_path, nullptr, false); if (!model) { diff --git a/cactus-engine/tests/asr.cpp b/cactus-engine/tests/transcribe.cpp similarity index 100% rename from cactus-engine/tests/asr.cpp rename to cactus-engine/tests/transcribe.cpp diff --git a/cactus-graph/CMakeLists.txt b/cactus-graph/CMakeLists.txt index e99787137..c82b62a02 100644 --- a/cactus-graph/CMakeLists.txt +++ b/cactus-graph/CMakeLists.txt @@ -23,6 +23,8 @@ add_library(cactus_graph STATIC src/ops_cache.cpp src/ops_dsp.cpp src/ops_image.cpp + src/graph_ffi.cpp + src/last_error.cpp ) set_target_properties(cactus_graph PROPERTIES POSITION_INDEPENDENT_CODE ON) @@ -40,7 +42,7 @@ target_compile_options(cactus_graph PRIVATE $<$:-Wno-pedantic> ) -if(BUILD_TESTING OR CACTUS_BUILD_TESTS) +if((BUILD_TESTING OR CACTUS_BUILD_TESTS) AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) file(GLOB TEST_SOURCES "tests/test_*.cpp") foreach(TEST_SRC ${TEST_SOURCES}) get_filename_component(TEST_NAME ${TEST_SRC} NAME_WE) diff --git a/cactus-graph/cactus_graph.h b/cactus-graph/cactus_graph.h index ffa0851cb..3d7db196e 100644 --- a/cactus-graph/cactus_graph.h +++ b/cactus-graph/cactus_graph.h @@ -806,4 +806,268 @@ namespace GraphFile { }; } +#if __GNUC__ >= 4 + #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) +#else + #define CACTUS_FFI_EXPORT +#endif + +extern std::string last_error_message; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* cactus_graph_t; +typedef uint64_t cactus_node_t; + +typedef struct { + int32_t precision; + size_t rank; + size_t shape[8]; + size_t num_elements; + size_t byte_size; +} cactus_tensor_info_t; + +CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); + +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); +CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); +CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); + +CACTUS_FFI_EXPORT int cactus_graph_input( + cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, +cactus_node_t* out_node); + +CACTUS_FFI_EXPORT int cactus_graph_set_input( + cactus_graph_t graph, cactus_node_t node, const void* data, int32_t +precision); +CACTUS_FFI_EXPORT int cactus_graph_set_external_input( + cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); +CACTUS_FFI_EXPORT int cactus_graph_mark_embedded_input( + cactus_graph_t graph, cactus_node_t node); + +CACTUS_FFI_EXPORT int cactus_graph_precision_cast( + cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, +cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_not_equal(cactus_graph_t graph, cactus_node_t +a, cactus_node_t b, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_not_equal(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, +float exponent, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_view( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_flatten( + cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, +cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_reshape( + cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose( + cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_transpose_n( + cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_slice( + cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_index( + cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_cumsum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_concat( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, +cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_cat( + cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t +axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_matmul( + cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gather( + cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( + cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( + cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( + cactus_graph_t graph, const char* filename, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_bind_mmap_weights( + cactus_graph_t graph, cactus_node_t node, const char* filename); +CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( + cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); +CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); + +CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_layernorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_groupnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_batchnorm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rms_norm( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( + cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( + cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, + const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, + size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_kv_cache_state( + cactus_graph_t graph, size_t max_seq_len, size_t num_kv_heads, size_t head_dim, size_t window_size, size_t sink_size, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_kv_cache_append( + cactus_graph_t graph, cactus_node_t new_kv, cactus_node_t cache_state, size_t window_size, size_t sink_size, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_attention_cached( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, + cactus_node_t k_cache_state, cactus_node_t v_cache_state, + float scale, size_t position_offset, size_t window_size, size_t v_head_dim, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv_cache_state( + cactus_graph_t graph, size_t window_size, size_t hidden_dim, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv_cache_append( + cactus_graph_t graph, cactus_node_t new_data, cactus_node_t cache_state, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv_cache_initialize( + cactus_graph_t graph, cactus_node_t rows, cactus_node_t cache_state, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_state( + cactus_graph_t graph, const size_t* shape, size_t shape_len, int precision, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_recurrent_cache_write( + cactus_graph_t graph, cactus_node_t new_value, cactus_node_t cache_input, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_rfft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_irfft( + cactus_graph_t graph, cactus_node_t input, size_t output_length, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_mel_filter_bank( + cactus_graph_t graph, size_t num_frequency_bins, size_t num_mel_filters, + float min_frequency, float max_frequency, size_t sampling_rate, + int norm_type, int scale_type, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_spectrogram( + cactus_graph_t graph, cactus_node_t waveform, cactus_node_t mel_filters, + size_t frame_length, size_t hop_length, size_t fft_length, + float power, bool center, int pad_mode, + float mel_floor, int log_mel_mode, + float dither, float preemphasis, bool remove_dc_offset, + cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_image_preprocess( + cactus_graph_t graph, cactus_node_t pixel_input, + int src_width, int src_height, int target_width, int target_height, + int patch_size, int channels, float rescale_factor, + const float* mean, const float* std_dev, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( + cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( + cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_stft( + cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_altup_predict( + cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_altup_correct( + cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( + cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( + cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, + const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, + size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); + +CACTUS_FFI_EXPORT int cactus_graph_sample( + cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( + cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_persistent( + cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); +CACTUS_FFI_EXPORT int cactus_graph_is_populated( + cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); +CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( + cactus_graph_t graph, cactus_node_t persistent_node); + +CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); +CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, +cactus_node_t node, void** out_ptr); +CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, +cactus_node_t node, cactus_tensor_info_t* out_info); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/cactus-graph/src/core.cpp b/cactus-graph/src/core.cpp index 7aff9cf49..1ad864bb9 100644 --- a/cactus-graph/src/core.cpp +++ b/cactus-graph/src/core.cpp @@ -87,7 +87,16 @@ BufferDesc::BufferDesc(BufferDesc&& other) noexcept num_groups(other.num_groups), activation_scales_data(other.activation_scales_data), owned_activation_scales(std::move(other.owned_activation_scales)), - num_rows_for_activation_scales(other.num_rows_for_activation_scales) { + num_rows_for_activation_scales(other.num_rows_for_activation_scales), + cq_codebook(other.cq_codebook), + cq_input_scale(other.cq_input_scale), + cq_input_scale_recip(other.cq_input_scale_recip), + cq_norms(other.cq_norms), + cq_left_signs(other.cq_left_signs), + cq_right_signs(other.cq_right_signs), + cq_permutation(other.cq_permutation), + cq_rotation(other.cq_rotation), + cq_flags(other.cq_flags) { other.total_size = 0; other.byte_size = 0; other.external_data = nullptr; @@ -96,6 +105,15 @@ BufferDesc::BufferDesc(BufferDesc&& other) noexcept other.num_groups = 0; other.activation_scales_data = nullptr; other.num_rows_for_activation_scales = 0; + other.cq_codebook = nullptr; + other.cq_input_scale = nullptr; + other.cq_input_scale_recip = nullptr; + other.cq_norms = nullptr; + other.cq_left_signs = nullptr; + other.cq_right_signs = nullptr; + other.cq_permutation = nullptr; + other.cq_rotation = nullptr; + other.cq_flags = 0; } BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { @@ -116,6 +134,15 @@ BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { activation_scales_data = other.activation_scales_data; owned_activation_scales = std::move(other.owned_activation_scales); num_rows_for_activation_scales = other.num_rows_for_activation_scales; + cq_codebook = other.cq_codebook; + cq_input_scale = other.cq_input_scale; + cq_input_scale_recip = other.cq_input_scale_recip; + cq_norms = other.cq_norms; + cq_left_signs = other.cq_left_signs; + cq_right_signs = other.cq_right_signs; + cq_permutation = other.cq_permutation; + cq_rotation = other.cq_rotation; + cq_flags = other.cq_flags; other.total_size = 0; other.byte_size = 0; @@ -125,6 +152,15 @@ BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { other.num_groups = 0; other.activation_scales_data = nullptr; other.num_rows_for_activation_scales = 0; + other.cq_codebook = nullptr; + other.cq_input_scale = nullptr; + other.cq_input_scale_recip = nullptr; + other.cq_norms = nullptr; + other.cq_left_signs = nullptr; + other.cq_right_signs = nullptr; + other.cq_permutation = nullptr; + other.cq_rotation = nullptr; + other.cq_flags = 0; } return *this; } diff --git a/cactus-engine/src/graph_ffi.cpp b/cactus-graph/src/graph_ffi.cpp similarity index 99% rename from cactus-engine/src/graph_ffi.cpp rename to cactus-graph/src/graph_ffi.cpp index 4d9b78c47..02861ab63 100644 --- a/cactus-engine/src/graph_ffi.cpp +++ b/cactus-graph/src/graph_ffi.cpp @@ -1,11 +1,7 @@ -#include "../cactus_engine.h" -#include "utils.h" -#include "cactus_graph.h" +#include "../cactus_graph.h" #include #include -using namespace cactus::ffi; - struct GraphHandle { CactusGraph graph; }; diff --git a/cactus-graph/src/last_error.cpp b/cactus-graph/src/last_error.cpp new file mode 100644 index 000000000..3810e9af2 --- /dev/null +++ b/cactus-graph/src/last_error.cpp @@ -0,0 +1,8 @@ +#include "../cactus_graph.h" +#include + +std::string last_error_message; + +extern "C" const char* cactus_get_last_error() { + return last_error_message.c_str(); +} diff --git a/cactus-graph/test.sh b/cactus-graph/test.sh index 1dc01db14..4bf78b6d8 100755 --- a/cactus-graph/test.sh +++ b/cactus-graph/test.sh @@ -3,19 +3,38 @@ set -e cd "$(dirname "$0")" +SUITE="" +while [[ $# -gt 0 ]]; do + case $1 in + --suite) SUITE="${2:?--suite needs an argument}"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; + esac +done + echo "Building and testing cactus-graph..." rm -rf build mkdir -p build cd build -cmake .. -DCACTUS_BUILD_TESTS=ON -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 +cmake .. -DCACTUS_BUILD_TESTS=ON -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) echo "" FAILED=0 -for test_bin in test_*; do - [ -x "$test_bin" ] && ./"$test_bin" || FAILED=1 -done +if [ -n "$SUITE" ]; then + target="./test_${SUITE}" + if [ -x "$target" ]; then + "$target" || FAILED=1 + else + echo "Test not found: $target" >&2 + FAILED=1 + fi +else + for t in ./test_*; do + [ -x "$t" ] || continue + "$t" || FAILED=1 + done +fi exit $FAILED diff --git a/cactus-kernels/CMakeLists.txt b/cactus-kernels/CMakeLists.txt index 5312512da..bd0f5ff03 100644 --- a/cactus-kernels/CMakeLists.txt +++ b/cactus-kernels/CMakeLists.txt @@ -27,20 +27,13 @@ set_target_properties(cactus_kernels PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(cactus_kernels PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/src - PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/libs + ${CMAKE_CURRENT_SOURCE_DIR}/libs ) target_compile_options(cactus_kernels PUBLIC -march=armv8.2-a+fp16+simd+dotprod+i8mm -pthread -O3 ) -target_compile_definitions(cactus_kernels PUBLIC - __ARM_NEON=1 - __ARM_FEATURE_FP16_VECTOR_ARITHMETIC=1 - __ARM_FEATURE_DOTPROD=1 - __ARM_FEATURE_MATMUL_INT8=1 -) - target_compile_options(cactus_kernels PRIVATE -Wall -Wextra -pedantic -Wno-missing-field-initializers $<$:-Wno-c99-extensions> @@ -50,20 +43,11 @@ target_compile_options(cactus_kernels PRIVATE if(APPLE) set(CMAKE_OSX_ARCHITECTURES "arm64") target_compile_definitions(cactus_kernels PUBLIC ACCELERATE_NEW_LAPACK) - find_library(ACCELERATE_FRAMEWORK Accelerate REQUIRED) - target_link_libraries(cactus_kernels PUBLIC ${ACCELERATE_FRAMEWORK}) + target_link_libraries(cactus_kernels PUBLIC "-framework Accelerate") endif() -if(BUILD_TESTING OR CACTUS_BUILD_TESTS) - set(TEST_SOURCES - tests/test_elementwise.cpp - tests/test_matmul.cpp - tests/test_attention.cpp - tests/test_reduce.cpp - tests/test_conv.cpp - tests/test_quant.cpp - tests/test_dsp.cpp - ) +if((BUILD_TESTING OR CACTUS_BUILD_TESTS) AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + file(GLOB TEST_SOURCES "tests/test_*.cpp") foreach(TEST_SRC ${TEST_SOURCES}) get_filename_component(TEST_NAME ${TEST_SRC} NAME_WE) add_executable(${TEST_NAME} ${TEST_SRC}) diff --git a/cactus-kernels/test.sh b/cactus-kernels/test.sh index df2d247d7..9c6d1ca01 100755 --- a/cactus-kernels/test.sh +++ b/cactus-kernels/test.sh @@ -3,19 +3,38 @@ set -e cd "$(dirname "$0")" +SUITE="" +while [[ $# -gt 0 ]]; do + case $1 in + --suite) SUITE="${2:?--suite needs an argument}"; shift 2 ;; + *) echo "Unknown arg: $1" >&2; exit 2 ;; + esac +done + echo "Building and testing cactus-kernels..." rm -rf build mkdir -p build cd build -cmake .. -DCACTUS_BUILD_TESTS=ON -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 +cmake .. -DCACTUS_BUILD_TESTS=ON -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) echo "" FAILED=0 -for test_bin in test_*; do - [ -x "$test_bin" ] && ./"$test_bin" || FAILED=1 -done +if [ -n "$SUITE" ]; then + target="./test_${SUITE}" + if [ -x "$target" ]; then + "$target" || FAILED=1 + else + echo "Test not found: $target" >&2 + FAILED=1 + fi +else + for t in ./test_*; do + [ -x "$t" ] || continue + "$t" || FAILED=1 + done +fi exit $FAILED diff --git a/cactus-kernels/tests/test_conv.cpp b/cactus-kernels/tests/test_conv.cpp index b7e97788c..42b916c1e 100644 --- a/cactus-kernels/tests/test_conv.cpp +++ b/cactus-kernels/tests/test_conv.cpp @@ -6,7 +6,7 @@ using namespace TestUtils; bool test_conv1d_k3() { const size_t N = 1, L = 32, C_in = 8, C_out = 16, stride = 1; - const size_t L_out = (L - 3) / stride + 1; + const size_t L_out = ((L - 1) / stride) + 1; std::vector<__fp16> input(N * L * C_in), weight(C_out * 3 * C_in), output(N * L_out * C_out); fill_random_fp16(input, -0.5f, 0.5f); fill_random_fp16(weight, -0.5f, 0.5f); @@ -89,7 +89,7 @@ bool test_maxpool1d() { bool run_benchmarks() { { const size_t N = 1, L = 3000, C_in = 80, C_out = 512, stride = 1; - const size_t L_out = (L - 3) / stride + 1; + const size_t L_out = ((L - 1) / stride) + 1; std::vector<__fp16> input(N * L * C_in), weight(C_out * 3 * C_in), output(N * L_out * C_out); fill_random_fp16(input, -0.5f, 0.5f); fill_random_fp16(weight, -0.5f, 0.5f); diff --git a/cactus-kernels/tests/test_elementwise.cpp b/cactus-kernels/tests/test_elementwise.cpp index e6f08e8f4..bae21e4e8 100644 --- a/cactus-kernels/tests/test_elementwise.cpp +++ b/cactus-kernels/tests/test_elementwise.cpp @@ -94,6 +94,21 @@ bool test_transpose_2d_f16() { return true; } +bool test_fast_tanh_f32x4() { + constexpr float TOL = 1e-5f; + for (int i = -10000; i <= 10000; ++i) { + float x = 0.001f * static_cast(i); + float32x4_t r = fast_tanh_f32x4(vdupq_n_f32(x)); + float lanes[4]; + vst1q_f32(lanes, r); + float want = std::tanh(x); + for (int k = 0; k < 4; ++k) { + if (std::fabs(lanes[k] - want) > TOL) return false; + } + } + return true; +} + bool run_benchmarks() { auto bench_binary = [](const char* label, void(*fn)(const __fp16*, const __fp16*, __fp16*, size_t)) { const size_t n = 1024 * 1024; @@ -168,6 +183,7 @@ int main() { runner.run_test("add_clipped_f16", test_add_clipped_f16()); runner.run_test("scalar_ops_f16", test_scalar_ops_f16()); runner.run_test("transpose_2d_f16", test_transpose_2d_f16()); + runner.run_test("fast_tanh_f32x4", test_fast_tanh_f32x4()); runner.print_benchmarks_header(); runner.run_bench("benchmarks", run_benchmarks()); runner.print_summary(); diff --git a/cactus-kernels/tests/test_matmul.cpp b/cactus-kernels/tests/test_matmul.cpp index 0eb443cdb..7e765b177 100644 --- a/cactus-kernels/tests/test_matmul.cpp +++ b/cactus-kernels/tests/test_matmul.cpp @@ -1,6 +1,7 @@ #include "test_utils.h" #include #include +#include #include using namespace TestUtils; diff --git a/cactus/CMakeLists.txt b/cactus/CMakeLists.txt deleted file mode 100644 index 41167424c..000000000 --- a/cactus/CMakeLists.txt +++ /dev/null @@ -1,57 +0,0 @@ -cmake_minimum_required(VERSION 3.10) -project(Cactus LANGUAGES CXX) - -set(CMAKE_BUILD_TYPE Release CACHE STRING "") -set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) - -if(NOT TARGET cactus_engine) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../cactus-engine ${CMAKE_CURRENT_BINARY_DIR}/cactus-engine) -endif() - -set(_ARCHIVES $ $ $) -set(_DEPS cactus_engine cactus_graph cactus_kernels) - -if(APPLE) - add_custom_target(cactus ALL - COMMAND libtool -static -o ${CMAKE_BINARY_DIR}/libcactus.a ${_ARCHIVES} - DEPENDS ${_DEPS} - ) -else() - file(GENERATE OUTPUT ${CMAKE_BINARY_DIR}/combine.mri CONTENT - "create ${CMAKE_BINARY_DIR}/libcactus.a\naddlib $\naddlib $\naddlib $\nsave\nend\n" - ) - add_custom_target(cactus ALL - COMMAND ${CMAKE_AR} -M < ${CMAKE_BINARY_DIR}/combine.mri - DEPENDS ${_DEPS} - ) -endif() - -if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) -file(WRITE ${CMAKE_BINARY_DIR}/stub.cpp "") -add_library(cactus_shared SHARED ${CMAKE_BINARY_DIR}/stub.cpp) -set_target_properties(cactus_shared PROPERTIES OUTPUT_NAME cactus) -add_dependencies(cactus_shared ${_DEPS}) - -if(APPLE) - target_link_options(cactus_shared PRIVATE - -Wl,-force_load,$ - -Wl,-force_load,$ - -Wl,-force_load,$ - ) - set(_CURL "${CACTUS_CURL_ROOT}/macos/libcurl.a") - if(EXISTS "${_CURL}") - target_link_libraries(cactus_shared PRIVATE "${_CURL}") - endif() - target_link_libraries(cactus_shared PRIVATE - "-framework Accelerate" "-framework CoreML" "-framework Foundation" - "-framework Security" "-framework SystemConfiguration" "-framework CFNetwork" - ) -else() - target_link_options(cactus_shared PRIVATE -Wl,--whole-archive ${_ARCHIVES} -Wl,--no-whole-archive) - find_package(CURL QUIET) - if(CURL_FOUND) - target_link_libraries(cactus_shared PRIVATE ${CURL_LIBRARIES}) - endif() -endif() -endif() diff --git a/cactus/build.sh b/cactus/build.sh deleted file mode 100755 index cd913a9d4..000000000 --- a/cactus/build.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -e - -missing=() -command -v cmake &> /dev/null || missing+=("cmake") -command -v make &> /dev/null || missing+=("make") -command -v g++ &> /dev/null || command -v clang++ &> /dev/null || missing+=("g++") - -if [ ${#missing[@]} -gt 0 ]; then - echo "Error: Missing required build tools: ${missing[*]}" - echo "" - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - echo "Install with: sudo apt-get install cmake build-essential" - elif [[ "$OSTYPE" == "darwin"* ]]; then - echo "Install with: xcode-select --install && brew install cmake" - fi - exit 1 -fi - -cd "$(dirname "$0")" - -echo "Building Cactus library..." - -LOCK_DIR="${TMPDIR:-/tmp}/cactus-build.lock" -while ! mkdir "$LOCK_DIR" 2>/dev/null; do - if [ -f "$LOCK_DIR/pid" ] && ! kill -0 "$(cat "$LOCK_DIR/pid" 2>/dev/null)" 2>/dev/null; then - rm -rf "$LOCK_DIR" - continue - fi - sleep 2 -done -echo "$$" > "$LOCK_DIR/pid" -trap 'rm -rf "$LOCK_DIR"' EXIT - -mkdir -p build -cd build - -cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF > /dev/null 2>&1 -make -j$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4) - -echo "Cactus library built successfully!" -echo " Static: $(pwd)/libcactus.a" -echo " Shared: $(pwd)/libcactus.$([ "$(uname)" = Darwin ] && echo dylib || echo so)" diff --git a/cactus/cactus.h b/cactus/cactus.h deleted file mode 100644 index 8d5032c5c..000000000 --- a/cactus/cactus.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef CACTUS_H -#define CACTUS_H - -#include "cactus_kernels.h" -#include "cactus_graph.h" -#include "cactus_engine.h" - -#endif // CACTUS_H diff --git a/cactus/engine/engine.h b/cactus/engine/engine.h deleted file mode 100644 index afebfeb07..000000000 --- a/cactus/engine/engine.h +++ /dev/null @@ -1,1104 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "../graph/graph.h" - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wc99-extensions" -#pragma clang diagnostic ignored "-Wunused-parameter" -#elif defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wunused-parameter" -#endif - -extern "C" { - #include "../../libs/stb/stb_image.h" - #include "../../libs/stb/stb_image_resize2.h" -} - -#ifdef __clang__ -#pragma clang diagnostic pop -#elif defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - -class CactusGraph; - -namespace cactus { -namespace npu { - class NPUPrefill; -} -namespace engine { - -class Siglip2Preprocessor; - -struct Config { - uint32_t vocab_size = 151936; - uint32_t bos_token_id = 151643; - uint32_t eos_token_id = 151645; - uint32_t num_layers = 28; - uint32_t hidden_dim = 1024; - uint32_t ffn_intermediate_dim = 3072; - uint32_t attention_heads = 16; - uint32_t attention_kv_heads = 8; - uint32_t attention_head_dim = 128; - float layer_norm_eps = 1e-6f; - float rope_theta = 1000000.0f; - uint32_t num_experts = 0; - uint32_t num_shared_experts = 0; - uint32_t num_top_experts = 0; - uint32_t moe_every_n_layers = 0; - uint32_t moe_intermediate_dim = 0; - uint32_t num_dense_layers = 0; - uint32_t num_experts_per_tok = 0; - bool norm_topk_prob = false; - bool use_expert_bias = false; - float routed_scaling_factor = 1.0f; - bool tie_word_embeddings = true; - - uint32_t vision_hidden_dim = 0; - uint32_t vision_num_layers = 0; - uint32_t vision_attention_heads = 0; - uint32_t vision_image_size = 0; - uint32_t vision_patch_size = 0; - uint32_t vision_num_channels = 3; - uint32_t vision_embed_dim = 0; - uint32_t visual_tokens_per_img = 0; - bool use_pixel_shuffle = false; - uint32_t pixel_shuffle_factor = 1; - bool use_image_tokens = false; - uint32_t image_token_id = 0; - bool use_layout_tags = false; - uint32_t image_seq_len = 64; - - uint32_t global_image_size = 2048; - uint32_t max_tile_size = 512; - float rescale_factor = 0.00392156862745098f; - float image_mean = 0.5f; - float image_std = 0.5f; - - uint32_t downsample_factor = 2; - uint32_t min_tiles = 2; - uint32_t max_tiles = 10; - bool use_thumbnail = true; - uint32_t min_image_tokens = 64; - uint32_t max_image_tokens = 256; - uint32_t max_num_patches = 1024; - uint32_t tile_size = 512; - float max_pixels_tolerance = 2.0f; - bool do_image_splitting = true; - bool encoder_act_gelu = false; - bool decoder_act_gelu = false; - uint32_t num_encoder_layers = 0; - uint32_t num_decoder_layers = 0; - float partial_rotary_factor = 0.0f; - uint32_t pad_token_id = 0; - uint32_t conv_kernel_size = 0; - uint32_t subsampling_conv_kernel_size = 0; - uint32_t subsampling_conv_stride = 0; - uint32_t subsampling_conv_channels = 0; - uint32_t subsampling_factor = 0; - uint32_t num_mel_bins = 80; - std::string encoder_hidden_act = "silu"; - uint32_t linear_num_key_heads = 0; - uint32_t linear_key_head_dim = 0; - uint32_t linear_num_value_heads = 0; - uint32_t linear_value_head_dim = 0; - uint32_t linear_q_proj_dim = 0; - uint32_t linear_k_proj_dim = 0; - uint32_t linear_v_proj_dim = 0; - - uint32_t kv_lora_rank = 0; - uint32_t q_lora_rank = 0; - uint32_t qk_head_dim = 0; - uint32_t qk_nope_head_dim = 0; - uint32_t qk_rope_head_dim = 0; - uint32_t v_head_dim = 0; - uint32_t rope_interleave = 0; - bool attention_bias = false; - float rope_scaling_factor = 1.0f; - float rope_mscale_all_dim = 0.0f; - - enum class ModelType {QWEN = 0, GEMMA = 1, NOMIC = 3, LFM2 = 5, SIGLIP2 = 6, WHISPER = 7, MOONSHINE = 8, SILERO_VAD = 9, PARAKEET = 10, QWEN3P5 = 11, PARAKEET_TDT = 12, GEMMA3N = 13, YOUTU = 14, GEMMA4 = 15, PYANNOTE = 16, WESPEAKER = 17, NEEDLE = 18}; - uint32_t predictor_hidden_dim = 0; - uint32_t predictor_num_layers = 0; - uint32_t tdt_joint_dim = 0; - uint32_t tdt_num_durations = 0; - uint32_t tdt_blank_id = 0; - std::vector tdt_durations; - - ModelType model_type = ModelType::QWEN; - - enum class ModelVariant {DEFAULT = 0, VLM = 1, EXTRACT = 2, RAG = 3}; - ModelVariant model_variant = ModelVariant::DEFAULT; - - enum class Activation {GELU = 0, SILU = 1}; - Activation activation = Activation::SILU; - - enum class Backend {CPU = 0, NPU = 1}; - Backend default_backend = Backend::CPU; - - enum class Precision {INT8 = 0, FP16 = 1, FP32 = 2}; - Precision precision = Precision::FP32; - - float default_temperature = 0.6f; - float default_top_p = 0.95f; - size_t default_top_k = 20; - float default_max_tps = -1.0f; - float default_cloud_handoff_threshold = 0.0f; - size_t default_rolling_entropy_window = 10; - - std::vector layer_types; - size_t conv_L_cache = 0; - - uint32_t altup_num_inputs = 4; - uint32_t laurel_rank = 64; - uint32_t hidden_size_per_layer_input = 256; - uint32_t num_kv_shared_layers = 0; - uint32_t sliding_window = 512; - float rope_local_base_freq = 10000.0f; - float final_logit_softcapping = 0.0f; - float global_partial_rotary_factor = 1.0f; - uint32_t expert_intermediate_size = 0; - uint32_t global_head_dim = 0; - uint32_t num_global_kv_heads = 0; - bool attention_k_eq_v = false; - bool enable_moe_block = false; - std::vector activation_sparsity_ppf; - - uint32_t vision_head_dim = 64; - uint32_t vision_kv_heads = 12; - uint32_t vision_intermediate_size = 3072; - uint32_t vision_position_embedding_size = 10240; - uint32_t vision_pooling_kernel_size = 3; - uint32_t vision_default_output_length = 280; - float vision_rope_theta = 100.0f; - - uint32_t audio_hidden_dim = 0; - uint32_t audio_num_layers = 0; - uint32_t audio_num_heads = 0; - uint32_t audio_head_dim = 0; - uint32_t audio_input_feat_size = 128; - uint32_t audio_conf_conv_kernel_size = 5; - uint32_t audio_chunk_size = 12; - uint32_t audio_context_left = 13; - uint32_t audio_context_right = 0; - float audio_logit_cap = 50.0f; - float audio_residual_weight = 0.5f; - uint32_t audio_output_proj_dims = 0; - uint32_t audio_vocab_size = 128; - uint32_t audio_vocab_offset = 0; - uint32_t audio_soft_tokens = 188; - uint32_t audio_sscp_conv0_channels = 128; - uint32_t audio_sscp_conv1_channels = 32; - float audio_sscp_conv_eps = 1e-3f; - float audio_rms_norm_eps = 1e-6f; - uint32_t audio_fft_length = 1024; - uint32_t audio_token_id = 0; - bool audio_fft_overdrive = false; - uint32_t channel_open_token_id = 100; - uint32_t channel_close_token_id = 101; - - static bool is_gemma_family(ModelType t) { - return t == ModelType::GEMMA || t == ModelType::GEMMA3N || t == ModelType::GEMMA4; - } - - bool from_json(const std::string& json_path); - std::string to_json() const; -}; - - - -struct MergeRule { - std::string first; - std::string second; - std::string merged; - uint32_t priority; - - MergeRule(const std::string& f, const std::string& s, const std::string& m, uint32_t p) - : first(f), second(s), merged(m), priority(p) {} -}; - - -struct ToolCallInfo { - std::string name; - std::string arguments; -}; - -struct ChatMessage { - std::string role; - std::string content; - std::string name; - std::vector images; - std::vector audio; - size_t audio_soft_token_count = 0; - std::vector tool_calls; -}; - -inline std::string format_needle_query_text(const std::vector& messages) { - std::string system_text; - std::string user_query; - - for (const auto& msg : messages) { - if (msg.role == "system") { - if (!system_text.empty()) { - system_text += "\n"; - } - system_text += msg.content; - } else if (msg.role == "user") { - user_query = msg.content; - } - } - - if (user_query.empty() && !messages.empty()) { - user_query = messages.back().content; - } - if (system_text.empty()) { - return user_query; - } - if (user_query.empty()) { - return system_text; - } - return system_text + "\n\n" + user_query; -} - -struct ToolConstraintSpec { - std::string name; - std::vector parameter_names; - std::vector required_parameter_names; -}; - -struct TokenizerRuntimeConfig { - enum class TokenizerType { UNKNOWN, BPE, SENTENCEPIECE }; - enum class VocabFormat { UNKNOWN, ID_TAB_TOKEN, LINE_TOKEN }; - enum class Normalizer { NONE, METASPACE, BYTE_LEVEL }; - enum class Decoder { NONE, REPLACE_METASPACE, BYTE_LEVEL }; - - TokenizerType tokenizer_type = TokenizerType::UNKNOWN; - VocabFormat vocab_format = VocabFormat::UNKNOWN; - Normalizer normalizer = Normalizer::NONE; - Decoder decoder = Decoder::NONE; - bool byte_fallback = false; - bool has_chat_template = false; -}; - -TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file); -void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens); -std::vector split_with_special_tokens(const std::string& text, const std::unordered_map& special_tokens); - -inline std::string extract_json_string(const std::string& json, size_t& pos) { - std::string value; - while (pos < json.size() && json[pos] != '"') { - if (json[pos] == '\\' && pos + 1 < json.size()) { - pos++; - if (json[pos] == 'n') value += '\n'; - else if (json[pos] == 't') value += '\t'; - else if (json[pos] == 'r') value += '\r'; - else if (json[pos] == '"') value += '"'; - else if (json[pos] == '\\') value += '\\'; - else value += json[pos]; - } else { - value += json[pos]; - } - pos++; - } - if (pos < json.size()) pos++; - return value; -} - -class Tokenizer { -public: - virtual ~Tokenizer() = default; - - virtual std::vector encode(const std::string& text) const = 0; - virtual std::string decode(const std::vector& tokens) const = 0; - - virtual std::vector apply_chat_template(const std::vector& messages, bool add_generation_prompt = true) const; - virtual std::string format_chat_prompt(const std::vector& messages, bool add_generation_prompt = true, const std::string& tools_json = "", bool enable_thinking_if_supported = false) const; - - virtual uint32_t get_vocab_size() const = 0; - virtual uint32_t get_unk_token() const = 0; - virtual uint32_t get_bos_token() const = 0; - virtual uint32_t get_eos_token() const = 0; - virtual bool has_chat_template() const { return has_chat_template_; } - std::string get_default_stop_sequence() const; - - virtual bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) = 0; - - uint32_t get_image_token_id() const { return image_token_id_; } - uint32_t get_fake_token_id() const { return fake_token_id_; } - uint32_t get_global_img_token_id() const { return global_img_token_id_; } - -protected: - enum class ModelType { UNKNOWN, QWEN, QWEN3P5, GEMMA, GEMMA4, LFM2, BERT, WHISPER, PARAKEET, YOUTU, NEEDLE}; - ModelType model_type_ = ModelType::UNKNOWN; - enum class ModelVariant { DEFAULT, VLM, EXTRACT, RAG}; - ModelVariant model_variant_ = ModelVariant::DEFAULT; - bool has_chat_template_ = false; - std::string chat_template_; - - uint32_t image_token_id_ = 396; - uint32_t fake_token_id_ = 49189; - uint32_t global_img_token_id_ = 49152; - - - uint32_t vision_patch_size_ = 16; - uint32_t vision_pooling_kernel_size_ = 3; - uint32_t vision_default_output_length_ = 280; - uint32_t vision_image_size_ = 768; - TokenizerRuntimeConfig runtime_config_; - - void detect_model_type(const std::string& config_path); - void load_chat_template(const std::string& template_file); - std::string format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; - std::string format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_gemma4_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported = false) const; - std::string format_lfm2_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_lfm2_vl_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_needle_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; - std::string format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const; -}; - -class BPETokenizer : public Tokenizer { -public: - BPETokenizer(); - ~BPETokenizer(); - - bool load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file); - bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; - - std::vector encode(const std::string& text) const override; - std::string decode(const std::vector& tokens) const override; - - uint32_t get_vocab_size() const override { return vocab_size_; } - uint32_t get_unk_token() const override { return unk_token_id_; } - uint32_t get_bos_token() const override { return bos_token_id_; } - uint32_t get_eos_token() const override { return eos_token_id_; } - -private: - std::unordered_map token_to_id_; - std::vector id_to_token_; - std::vector merge_rules_; - std::unordered_map merge_map_; - - uint32_t vocab_size_; - uint32_t unk_token_id_; - uint32_t bos_token_id_; - uint32_t eos_token_id_; - - void* vocab_mmap_ptr_; - size_t vocab_mmap_size_; - - void* merges_mmap_ptr_; - size_t merges_mmap_size_; - - std::vector apply_bpe(const std::vector& tokens) const; - std::pair find_best_merge_fast(const std::vector& tokens) const; - - std::string bytes_to_unicode(const std::string& text) const; - std::string unicode_to_bytes(const std::string& text) const; - std::vector byte_level_split(const std::string& text) const; - std::vector utf8_split(const std::string& text) const; - - void cleanup_mmap(); - -private: - mutable std::unordered_map byte_to_unicode_; - mutable std::unordered_map unicode_to_byte_; - void init_byte_mappings() const; - - std::unordered_map special_tokens_; - std::vector split_with_special_tokens(const std::string& text) const; - void load_special_tokens(const std::string& config_file); -}; - -class SPTokenizer : public Tokenizer { -public: - SPTokenizer(); - ~SPTokenizer(); - - bool load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) override; - - std::vector encode(const std::string& text) const override; - std::string decode(const std::vector& tokens) const override; - - uint32_t get_vocab_size() const override { return vocab_size_; } - uint32_t get_unk_token() const override { return unk_token_id_; } - uint32_t get_bos_token() const override { return bos_token_id_; } - uint32_t get_eos_token() const override { return eos_token_id_; } - -private: - struct TrieNode { - std::unordered_map> children; - int32_t token_id = -1; - float score = 0.0f; - }; - - std::unique_ptr trie_root_; - std::unordered_map token_to_id_; - std::vector id_to_token_; - std::vector token_scores_; - - uint32_t vocab_size_; - uint32_t unk_token_id_; - uint32_t bos_token_id_; - uint32_t eos_token_id_; - uint32_t pad_token_id_; - - bool sp_bpe_mode_ = false; - bool sp_add_dummy_prefix_ = false; - bool sp_byte_fallback_ = false; - - void* vocab_mmap_ptr_; - size_t vocab_mmap_size_; - - void build_trie(); - std::vector> tokenize_with_trie(const std::string& text) const; - std::vector tokenize_with_bpe(const std::string& text) const; - std::string preprocess_text(const std::string& text) const; - std::string postprocess_text(const std::string& text) const; - std::vector split_by_unicode_spaces(const std::string& text) const; - - void cleanup_mmap(); - - std::unordered_map special_tokens_; - std::vector split_with_special_tokens(const std::string& text) const; - void load_special_tokens(const std::string& config_file); -}; - -class ConvCache { -public: - struct CircularView { - const void* ptr1; - size_t len1; - const void* ptr2; - size_t len2; - size_t total_len; - }; - - void init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision); - CircularView get_window(size_t layer) const; - void update(CactusGraph* gb, size_t layer, const size_t latest_token); - void reset(); - - bool is_empty() const { return num_layers == 0; } - - size_t num_layers = 0; - size_t hidden_size = 0; - size_t window_size = 0; - Precision precision = Precision::FP32; - size_t element_size = 4; - -private: - struct LayerState { - std::vector data; - size_t head = 0; - size_t count = 0; - }; - - std::vector layer_states; -}; - -struct KVCache { - static constexpr size_t DEFAULT_WINDOW_SIZE = 1024; - static constexpr size_t DEFAULT_SINK_SIZE = 4; - - struct LayerCache { - std::vector keys; - std::vector values; - std::vector key_scales; - std::vector value_scales; - size_t head_dim = 0; - size_t kv_heads = 0; - }; - - std::vector layer_caches; - - size_t window_size = DEFAULT_WINDOW_SIZE; - size_t sink_size = DEFAULT_SINK_SIZE; - size_t current_seq_len = 0; - size_t total_seq_len = 0; - size_t max_seq_len = 2048; - size_t num_layers = 0; - Precision precision; - size_t element_size = 4; - - void set_window_size(size_t window, size_t sink = DEFAULT_SINK_SIZE); - size_t get_effective_seq_len() const { return current_seq_len; } - size_t get_total_seq_len() const { return total_seq_len; } - size_t get_layer_head_dim(size_t layer_idx) const { return layer_caches[layer_idx].head_dim; } - size_t get_layer_kv_heads(size_t layer_idx) const { return layer_caches[layer_idx].kv_heads; } - - void init(size_t num_layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision); - void reset(); - void update_from_graph(CactusGraph* gb, const std::vector& k_nodes, - const std::vector& v_nodes, size_t seq_len, - size_t num_layers); - - void update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, - size_t num_tokens, size_t kv_heads, size_t head_dim); - - bool is_empty() const { return current_seq_len == 0; } - void* get_key_ptr(size_t layer); - void* get_value_ptr(size_t layer); - - struct CircularView { - const void* ptr1; - const void* ptr2; - size_t len1; - size_t len2; - size_t total_len; - }; - - CircularView get_key_view(size_t layer); - CircularView get_value_view(size_t layer); - - const int8_t* get_keys_int8(size_t layer) const; - const int8_t* get_values_int8(size_t layer) const; - const float* get_key_scales(size_t layer) const; - const float* get_value_scales(size_t layer) const; - - void remove_token_range(size_t start, size_t count); - void compact_to_windows(const std::vector& target_windows); -}; - -class ToolCallConstrainer { -public: - enum class State { - DONE, - - QWEN_START, - QWEN_EXPECT_OPEN_BRACE, - QWEN_EXPECT_NAME_KEY, - QWEN_EXPECT_NAME_COLON, - QWEN_EXPECT_NAME_VALUE, - QWEN_EXPECT_COMMA, - QWEN_EXPECT_ARGS_KEY, - QWEN_EXPECT_ARGS_COLON, - QWEN_IN_ARGUMENTS, - QWEN_EXPECT_CLOSE_BRACE, - QWEN_EXPECT_END, - - NEEDLE_START, - - LFM_START, - LFM_EXPECT_BRACKET, - LFM_IN_FUNC_NAME, - LFM_EXPECT_PAREN, - LFM_IN_ARGUMENTS, - LFM_EXPECT_BRACKET_CLOSE, - LFM_EXPECT_END, - - GEMMA_START, - GEMMA_EXPECT_CALL, - GEMMA_IN_FUNC_NAME, - GEMMA_EXPECT_BRACE, - GEMMA_IN_ARGUMENTS, - GEMMA_EXPECT_END - }; - - void init(Config::ModelType model_type, - const std::vector& tools, - Tokenizer* tokenizer); - - const std::unordered_map& get_bias() const { return current_bias_; } - - void update(uint32_t token_id, const std::string& decoded_text); - - void reset(); - - bool is_active() const { return active_; } - -private: - bool active_ = false; - State state_ = State::QWEN_START; - Config::ModelType model_type_ = Config::ModelType::QWEN; - Tokenizer* tokenizer_ = nullptr; - - bool is_gemma_family() const { return Config::is_gemma_family(model_type_); } - bool is_needle() const { return model_type_ == Config::ModelType::NEEDLE; } - - enum class NeedleJsonState { - FREE, - IN_NAME, - IN_ARG_KEY, - }; - - struct NeedleTrieNode { - std::unordered_map> children; - bool is_terminal = false; - }; - - std::vector tool_specs_; - std::vector function_names_; - std::string generated_text_; - int brace_depth_ = 0; - - std::string call_start_tag_; - std::string call_end_tag_; - - std::unordered_set qwen_tool_call_start_tokens_; - std::unordered_set qwen_tool_call_end_tokens_; - std::unordered_set open_brace_tokens_; - std::unordered_set close_brace_tokens_; - std::unordered_set colon_tokens_; - std::unordered_set comma_tokens_; - std::unordered_set name_key_tokens_; - std::unordered_set args_key_tokens_; - std::unordered_set quote_tokens_; - std::unordered_set backtick_tokens_; - std::unordered_set all_func_name_tokens_; - std::unordered_map> func_name_sequences_; - NeedleJsonState needle_json_state_ = NeedleJsonState::FREE; - std::string needle_buffer_; - std::string needle_constrained_buf_; - std::string needle_current_function_; - bool needle_in_arguments_ = false; - int needle_arguments_depth_ = 0; - int needle_nesting_depth_ = 0; - bool needle_in_string_value_ = false; - bool needle_prev_char_escape_ = false; - std::unique_ptr needle_name_trie_; - std::unordered_map> needle_param_tries_; - std::vector needle_token_strings_; - std::unordered_map> needle_token_index_; - - std::unordered_set tool_start_tokens_; - std::unordered_set tool_end_tokens_; - std::unordered_set bracket_open_tokens_; - std::unordered_set bracket_close_tokens_; - std::unordered_set paren_open_tokens_; - std::unordered_set paren_close_tokens_; - std::unordered_set equals_tokens_; - - std::string lfm_current_function_; - std::string lfm_args_buffer_; - std::unordered_set lfm_seen_arg_keys_; - std::unordered_map> lfm_required_params_; - std::unordered_map> lfm_all_params_; - - std::unordered_set gemma_call_start_tokens_; - std::unordered_set gemma_call_end_tokens_; - std::unordered_set gemma_response_start_tokens_; - std::unordered_set gemma_call_prefix_tokens_; - std::unordered_set escape_tokens_; - - std::unordered_map current_bias_; - - void compute_bias(); - void tokenize_grammar_elements(); - void add_tokens_for_string(const std::string& str, std::unordered_set& token_set); - void add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set); - void add_tokens_containing(char needle, std::unordered_set& token_set); - void tokenize_function_names(bool quote_names); - void init_common_tokens(); - void init_needle_constraints(); - void reset_needle_constraints(); - void feed_needle_text(const std::string& text); - void feed_needle_char(char ch); - bool needle_at_arg_key_start() const; - bool needle_is_value_string_start() const; - void needle_insert_word(NeedleTrieNode* root, const std::string& word); - const NeedleTrieNode* needle_get_trie_node(const NeedleTrieNode* root, const std::string& prefix) const; - bool needle_check_token_valid(const std::string& token_text, const NeedleTrieNode* trie_node) const; -}; - -class Model { -public: - struct DebugNode { - uint32_t layer_idx; - std::string name; - size_t node_id; - }; - - Model(); - explicit Model(const Config& config); - virtual ~Model(); - - const Config& get_config() const { return config_; } - Tokenizer* get_tokenizer() const { return tokenizer_.get(); } - const std::vector& get_debug_nodes() const; - - virtual bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true); - - virtual bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true); - - virtual uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - virtual void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = ""); - - virtual void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = ""); - - virtual uint32_t decode_with_images(const std::vector& tokens, const std::vector& image_paths, - float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - virtual uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, float temperature = 0.0f, float top_p = 0.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr); - - std::vector get_embeddings(const std::vector& tokens, bool pooled = true, bool normalize = false, const std::string& profile_file = ""); - - virtual std::vector get_image_embeddings(const std::string& image_path); - - virtual std::vector get_audio_embeddings(const std::vector& audio_features); - - virtual void reset_cache() { kv_cache_.reset(); token_history_.clear(); } - void record_sampled_token(uint32_t token) { - if (token_history_.size() >= MAX_TOKEN_HISTORY) { - token_history_.erase(token_history_.begin(), token_history_.begin() + (MAX_TOKEN_HISTORY / 2)); - } - token_history_.push_back(token); - } - - double score_tokens_window_logprob(const std::vector& tokens, size_t start, size_t end, size_t context, size_t* tokens_scored); - - - - void set_cache_window(size_t window_size, size_t sink_size = 4) { kv_cache_.set_window_size(window_size, sink_size); } - size_t get_cache_size() const { return kv_cache_.current_seq_len; } - - bool load_npu_prefill(const std::string& model_path); - bool has_npu_prefill() const; - size_t get_prefill_chunk_size() const; - - virtual void remove_thinking_tokens(const std::vector>& ranges); - virtual void compact_kv_cache() {} - - virtual void set_tool_constraints(const std::vector& tools); - virtual void clear_tool_constraints(); - virtual void update_tool_constraints(uint32_t token_id); - - void* graph_handle_; - - void set_vocab_bias(const std::unordered_map& bias) { - vocab_bias_ = bias; - } - - void clear_vocab_bias() { - vocab_bias_.clear(); - } - - bool has_vocab_bias() const { - return !vocab_bias_.empty(); - } - - const std::unordered_map& get_vocab_bias() const { - return vocab_bias_; - } - -protected: - size_t sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, - float min_p, float repetition_penalty, - const std::unordered_map* extra_bias = nullptr) const; - - static void compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy); - - virtual size_t forward(const std::vector& tokens, bool use_cache = false) = 0; - - virtual size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false); - - virtual void load_weights_to_graph(CactusGraph* gb) = 0; - - virtual size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; - - virtual size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const = 0; - virtual size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) = 0; - void update_kv_cache(CactusGraph* gb, size_t seq_len); - virtual std::vector get_kv_layer_dims() const { - return std::vector(config_.num_layers, config_.attention_head_dim); - } - virtual std::vector get_kv_layer_heads() const { - return std::vector(config_.num_layers, config_.attention_kv_heads); - } - virtual void post_init() {} - virtual void post_execute_updates(CactusGraph*, size_t) {} - Config config_; - std::unique_ptr tokenizer_; - - bool initialized_; - float attention_scale_; - -protected: - KVCache kv_cache_; - std::vector cache_k_output_nodes_; - std::vector cache_v_output_nodes_; - - std::string embedding_file_path_; - size_t embedding_node_id_; - std::string model_folder_path_; - size_t output_weight_node_id_; - - mutable std::vector debug_nodes_; - - void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const; - void clear_debug_nodes(); - - bool init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup); - bool owns_graph_; - - std::unique_ptr npu_prefill_; - void prefill_npu(const std::vector& tokens); - virtual std::vector<__fp16> get_token_embeddings(const std::vector& tokens); - - static constexpr size_t MAX_TOKEN_HISTORY = 128; - ToolCallConstrainer tool_constrainer_; - std::vector token_history_; - -private: - std::unordered_map vocab_bias_; -}; - -std::unique_ptr create_model(const std::string& model_folder); - -class Siglip2Preprocessor { -public: - struct Config { - int patch_size = 16; - int downsample_factor = 2; - int min_tiles = 2; - int max_tiles = 10; - bool use_thumbnail = true; - int min_image_tokens = 64; - int max_image_tokens = 256; - int max_num_patches = 1024; - int tile_size = 512; - float max_pixels_tolerance = 2.0f; - bool do_resize = true; - bool do_rescale = true; - bool do_normalize = true; - bool do_convert_rgb = true; - bool do_image_splitting = true; - float rescale_factor = 1.0f / 255.0f; - float image_mean[3] = {0.5f, 0.5f, 0.5f}; - float image_std[3] = {0.5f, 0.5f, 0.5f}; - }; - - struct PreprocessedImage { - std::vector pixel_values; - std::vector pixel_attention_mask; - std::vector> spatial_shapes; - std::vector pixel_values_shape; - std::vector pixel_attention_mask_shape; - std::vector spatial_shapes_shape; - int num_patches_height; - int num_patches_width; - int actual_num_patches; - int num_tiles; - int patch_dim; - int max_patches_per_tile; - - int image_rows; - int image_cols; - int image_height; - int image_width; - int tokens_per_tile; - int thumbnail_tokens; - - ~PreprocessedImage(); - }; - - struct SpatialShapeResult { - std::vector> shapes; - int grid_rows; - int grid_cols; - }; - - explicit Siglip2Preprocessor(const Config& config); - Siglip2Preprocessor(); - ~Siglip2Preprocessor(); - - PreprocessedImage preprocess_from_file(const std::string& image_path); - PreprocessedImage preprocess_from_memory(const unsigned char* img_data, int width, int height, int channels); - SpatialShapeResult compute_spatial_shapes(int height, int width); - -private: - Config config_; - - std::pair compute_pixel_limits() const; - std::vector convert_to_rgb(const unsigned char* img_data, int width, int height, int channels); - std::pair smart_resize(int height, int width); - bool is_image_too_large(int height, int width); - std::pair get_grid_layout(int height, int width); - std::pair find_closest_aspect_ratio(float aspect_ratio, int width, int height); - std::vector resize_image(const unsigned char* img_data, int src_width, int src_height, - int dst_width, int dst_height, int channels); - std::vector normalize_image(const float* img_data, int width, int height, int channels); - std::vector> convert_image_to_patches( - const std::vector& image, int width, int height, int channels, int patch_size); - PreprocessedImage pad_patches(const std::vector>& tile_patches, - const std::vector>& spatial_shapes, - int patch_dim, - int max_patches_per_tile); - int round_by_factor(int number, int factor); -}; - -class AudioProcessor { -public: - struct SpectrogramConfig { - size_t n_fft = 400; - size_t hop_length = 160; - size_t frame_length = 400; - float power = 2.0f; - bool center = true; - const char* pad_mode = "reflect"; - bool onesided = true; - float dither = 0.0f; - float mel_floor = 1e-10f; - const char* log_mel = nullptr; - float reference = 1.0f; - float min_value = 1e-10f; - bool remove_dc_offset = false; - float preemphasis = 0.0f; - bool hann_periodic = true; - float window_a0 = 0.5f; - size_t fft_override = 0; - bool mel_floor_additive = false; - }; - - AudioProcessor(); - ~AudioProcessor(); - - void init_mel_filters(size_t num_frequency_bins, size_t num_mel_filters, - float min_freq, float max_freq, size_t sampling_rate, - const char* norm = "slaney", const char* mel_scale = "slaney"); - - std::vector compute_spectrogram( - const std::vector& waveform, - const SpectrogramConfig& config); - - static std::vector compute_irfft( - const std::vector& complex_input, - size_t n, - const char* norm = "backward"); - - const std::vector& get_mel_filters() const { return mel_filters_; } - - size_t get_num_mel_filters() const { return num_mel_filters_; } - size_t get_num_frequency_bins() const { return num_frequency_bins_; } - -private: - std::vector mel_filters_; - size_t num_frequency_bins_; - size_t num_mel_filters_; -}; - -namespace index { - constexpr uint32_t MAGIC = 0x43414354; - constexpr uint32_t VERSION = 1; - - struct Document { - int id; - std::vector embedding; - std::string content; - std::string metadata; - }; - - struct QueryResult { - int doc_id; - float score; - - QueryResult(int doc_id, float score) : doc_id(doc_id), score(score) {} - }; - - struct QueryOptions { - size_t top_k = 10; - float score_threshold = -1.0f; - }; - - class Index { - public: - Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim); - ~Index(); - - Index(const Index&) = delete; - Index& operator=(const Index&) = delete; - Index(Index&&) = delete; - Index& operator=(Index&&) = delete; - - void add_documents(const std::vector& documents); - void delete_documents(const std::vector& doc_ids); - std::vector get_documents(const std::vector& doc_ids); - std::vector> query(const std::vector>& embeddings, const QueryOptions& options); - void compact(); - - private: - struct IndexHeader { - uint32_t magic; - uint32_t version; - uint32_t embedding_dim; - uint32_t num_documents; - }; - - struct IndexEntry { - int32_t doc_id; - uint64_t data_offset; - uint8_t flags; // bit 0: tombstone - - const __fp16* embedding() const { - return reinterpret_cast(this + 1); - } - - static size_t size(size_t embedding_dim) { - return sizeof(IndexEntry) + embedding_dim * sizeof(__fp16); - } - }; - - struct DataHeader { - uint32_t magic; - uint32_t version; - }; - - struct DataEntry { - uint16_t content_len; - uint16_t metadata_len; - - const char* content() const { - return reinterpret_cast(this + 1); - } - - const char* metadata() const { - return content() + content_len; - } - }; - - void parse_index_header(); - void parse_data_header(); - void build_doc_id_map(); - void validate_documents(const std::vector& documents); - void validate_doc_ids(const std::vector& doc_ids); - ssize_t write_full(int fd, const void* buf, size_t count); - - std::unordered_map doc_id_map_; - - std::string index_path_, data_path_; - size_t embedding_dim_; - size_t index_entry_size_; - uint32_t num_documents_; - - int index_fd_, data_fd_; - void *mapped_index_, *mapped_data_; - size_t index_file_size_, data_file_size_; - }; -} // namespace index - -} -} diff --git a/cactus/engine/engine_bpe.cpp b/cactus/engine/engine_bpe.cpp deleted file mode 100644 index 58785735d..000000000 --- a/cactus/engine/engine_bpe.cpp +++ /dev/null @@ -1,566 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -namespace { -constexpr const char* kMetaspace = "\xE2\x96\x81"; -} // namespace - -BPETokenizer::BPETokenizer() - : vocab_size_(0), unk_token_id_(0), bos_token_id_(1), eos_token_id_(2), - vocab_mmap_ptr_(nullptr), vocab_mmap_size_(0), - merges_mmap_ptr_(nullptr), merges_mmap_size_(0) { - has_chat_template_ = false; -} - -BPETokenizer::~BPETokenizer() { - cleanup_mmap(); -} - -void BPETokenizer::cleanup_mmap() { - if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { - munmap(vocab_mmap_ptr_, vocab_mmap_size_); - vocab_mmap_ptr_ = nullptr; - } - if (merges_mmap_ptr_ && merges_mmap_ptr_ != MAP_FAILED) { - munmap(merges_mmap_ptr_, merges_mmap_size_); - merges_mmap_ptr_ = nullptr; - } -} - -bool BPETokenizer::load_vocabulary_mmap(const std::string& vocab_file, const std::string& merges_file) { - int vocab_fd = open(vocab_file.c_str(), O_RDONLY); - if (vocab_fd == -1) return false; - - struct stat vocab_stat; - if (fstat(vocab_fd, &vocab_stat) == -1) { - close(vocab_fd); - return false; - } - - vocab_mmap_size_ = vocab_stat.st_size; - vocab_mmap_ptr_ = mmap(nullptr, vocab_mmap_size_, PROT_READ, MAP_PRIVATE, vocab_fd, 0); - close(vocab_fd); - - if (vocab_mmap_ptr_ == MAP_FAILED) return false; - - std::string vocab_content(static_cast(vocab_mmap_ptr_), vocab_mmap_size_); - std::istringstream vocab_stream(vocab_content); - - auto rtrim_cr = [](std::string& s) { - if (!s.empty() && s.back() == '\r') s.pop_back(); - }; - - std::string line; - token_to_id_.clear(); - id_to_token_.clear(); - special_tokens_.clear(); - const bool use_id_tab_vocab = - runtime_config_.vocab_format == TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; - - if (use_id_tab_vocab) { - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - - std::string token; - uint32_t id = UINT32_MAX; - - std::istringstream iss(line); - if (iss >> id) { - if (std::getline(iss, token) && !token.empty() && token[0] == '\t') { - token = token.substr(1); - } - - if (token.empty()) { - auto last_pos = vocab_stream.tellg(); - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - if (!line.empty()) { - break; - } - token += '\n'; - last_pos = vocab_stream.tellg(); - } - vocab_stream.seekg(last_pos); - } - } - - if (!token.empty() && id != UINT32_MAX) { - token_to_id_[token] = id; - if (id >= id_to_token_.size()) { - id_to_token_.resize(id + 1); - } - id_to_token_[id] = token; - } - } - vocab_size_ = static_cast(id_to_token_.size()); - } else { - uint32_t id = 0; - while (std::getline(vocab_stream, line)) { - rtrim_cr(line); - token_to_id_[line] = id; - id_to_token_.push_back(line); - ++id; - } - vocab_size_ = id; - } - - int merges_fd = open(merges_file.c_str(), O_RDONLY); - if (merges_fd == -1) return false; - - struct stat merges_stat; - if (fstat(merges_fd, &merges_stat) == -1) { - close(merges_fd); - return false; - } - - merges_mmap_size_ = merges_stat.st_size; - merges_mmap_ptr_ = mmap(nullptr, merges_mmap_size_, PROT_READ, MAP_PRIVATE, merges_fd, 0); - close(merges_fd); - - if (merges_mmap_ptr_ == MAP_FAILED) return false; - - std::string merges_content(static_cast(merges_mmap_ptr_), merges_mmap_size_); - std::istringstream merges_stream(merges_content); - - merge_rules_.clear(); - uint32_t priority = 0; - - while (std::getline(merges_stream, line)) { - rtrim_cr(line); - if (line.empty() || line[0] == '#') continue; - - std::istringstream iss(line); - std::string first, second; - if (iss >> first >> second) { - rtrim_cr(first); - rtrim_cr(second); - - std::string merged = first + second; - merge_rules_.emplace_back(first, second, merged, priority); - - std::string key = first + "\x00" + second; - auto it = merge_map_.find(key); - if (it == merge_map_.end() || priority < it->second) { - merge_map_[key] = priority; - } - priority++; - } - } - - auto is_whitespace_only = [](const std::string& s) { - if (s.empty()) return false; - for (char c : s) { - if (c != ' ' && c != '\n' && c != '\t' && c != '\r') return false; - } - return true; - }; - std::vector ws_tokens; - for (const auto& tok : id_to_token_) { - if (tok.size() >= 2 && is_whitespace_only(tok)) ws_tokens.push_back(tok); - } - std::sort(ws_tokens.begin(), ws_tokens.end(), - [](const std::string& a, const std::string& b) { return a.size() < b.size(); }); - for (const auto& tok : ws_tokens) { - for (size_t i = 1; i < tok.size(); i++) { - std::string first = tok.substr(0, i); - std::string second = tok.substr(i); - if (!token_to_id_.count(first) || !token_to_id_.count(second)) continue; - std::string key = first + "\x00" + second; - if (merge_map_.find(key) == merge_map_.end()) { - merge_rules_.emplace_back(first, second, tok, priority); - merge_map_[key] = priority; - priority++; - } - break; - } - } - - std::sort(merge_rules_.begin(), merge_rules_.end(), - [](const MergeRule& a, const MergeRule& b) { - return a.priority < b.priority; - }); - - return true; -} - -bool BPETokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& merges_file, const std::string& config_file) { - runtime_config_ = load_tokenizer_runtime_config(config_file); - if (!load_vocabulary_mmap(vocab_file, merges_file)) { - return false; - } - - std::ifstream config_stream(config_file); - if (!config_stream.is_open()) { - return true; - } - - std::string line; - while (std::getline(config_stream, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = line.substr(0, eq_pos); - std::string value = line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "eos_token_id") { - eos_token_id_ = std::stoul(value); - } else if (key == "pad_token_id") { - if (unk_token_id_ == 0) { - unk_token_id_ = std::stoul(value); - } - } else if (key == "unk_token_id" && value != "null") { - unk_token_id_ = std::stoul(value); - } else if (key == "bos_token_id" && value != "null") { - bos_token_id_ = std::stoul(value); - } else if (key == "vocab_size") { - if (std::stoul(value) != vocab_size_) { - } - } - } - - std::string dir = config_file.substr(0, config_file.find_last_of("/\\")); - std::string special_tokens_path = dir + "/special_tokens.json"; - load_special_tokens(special_tokens_path); - - std::string template_path = dir + "/chat_template.jinja2"; - load_chat_template(template_path); - - std::string config_path = dir + "/config.txt"; - detect_model_type(config_path); - - return true; -} - -void BPETokenizer::load_special_tokens(const std::string& config_file) { - load_special_tokens_map(config_file, special_tokens_); -} - -std::vector BPETokenizer::split_with_special_tokens(const std::string& text) const { - return cactus::engine::split_with_special_tokens(text, special_tokens_); -} - -void BPETokenizer::init_byte_mappings() const { - if (!byte_to_unicode_.empty()) return; - - std::vector bytes; - - for (int i = 33; i <= 126; ++i) { - bytes.push_back(i); - } - - - for (int i = 161; i <= 255; ++i) { - bytes.push_back(i); - } - - std::vector remaining_bytes; - for (int i = 0; i <= 32; ++i) remaining_bytes.push_back(i); - remaining_bytes.push_back(127); - for (int i = 128; i <= 160; ++i) remaining_bytes.push_back(i); - - int unicode_start = 256; - for (int byte : remaining_bytes) { - bytes.push_back(byte); - } - - for (size_t i = 0; i < bytes.size(); ++i) { - uint8_t byte = static_cast(bytes[i]); - - if (byte >= 33 && byte <= 126) { - std::string unicode_char(1, static_cast(byte)); - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } else if (byte >= 161 && byte <= 255) { - std::string unicode_char; - unicode_char += static_cast(0xC0 | (byte >> 6)); - unicode_char += static_cast(0x80 | (byte & 0x3F)); - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } else { - int unicode_point = unicode_start++; - std::string unicode_char; - if (unicode_point < 0x800) { - unicode_char += static_cast(0xC0 | (unicode_point >> 6)); - unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); - } else { - unicode_char += static_cast(0xE0 | (unicode_point >> 12)); - unicode_char += static_cast(0x80 | ((unicode_point >> 6) & 0x3F)); - unicode_char += static_cast(0x80 | (unicode_point & 0x3F)); - } - byte_to_unicode_[byte] = unicode_char; - unicode_to_byte_[unicode_char] = byte; - } - } -} - -std::string BPETokenizer::bytes_to_unicode(const std::string& text) const { - init_byte_mappings(); - - std::string result; - for (uint8_t byte : text) { - result += byte_to_unicode_.at(byte); - } - return result; -} - -std::string BPETokenizer::unicode_to_bytes(const std::string& text) const { - init_byte_mappings(); - - std::string result; - size_t i = 0; - while (i < text.length()) { - std::string unicode_char; - - if ((text[i] & 0x80) == 0) { - unicode_char = text.substr(i, 1); - i += 1; - } else if ((text[i] & 0xE0) == 0xC0) { - unicode_char = text.substr(i, 2); - i += 2; - } else if ((text[i] & 0xF0) == 0xE0) { - unicode_char = text.substr(i, 3); - i += 3; - } else { - unicode_char = text.substr(i, 1); - i += 1; - } - - auto it = unicode_to_byte_.find(unicode_char); - if (it != unicode_to_byte_.end()) { - result += static_cast(it->second); - } else if (unicode_char.size() == 1 && (unsigned char)unicode_char[0] < 128) { - result += unicode_char[0]; - } else { - result += '?'; - } - } - return result; -} - -std::vector BPETokenizer::byte_level_split(const std::string& text) const { - std::string unicode_text = bytes_to_unicode(text); - - std::vector chars; - size_t i = 0; - while (i < unicode_text.length()) { - size_t char_len = 1; - - if ((unicode_text[i] & 0x80) == 0) { - char_len = 1; - } else if ((unicode_text[i] & 0xE0) == 0xC0) { - char_len = 2; - } else if ((unicode_text[i] & 0xF0) == 0xE0) { - char_len = 3; - } else if ((unicode_text[i] & 0xF8) == 0xF0) { - char_len = 4; - } - - if (i + char_len <= unicode_text.length()) { - chars.push_back(unicode_text.substr(i, char_len)); - } - i += char_len; - } - - return chars; -} - -std::vector BPETokenizer::utf8_split(const std::string& text) const { - std::vector chars; - size_t i = 0; - while (i < text.length()) { - size_t char_len = 1; - const unsigned char byte = static_cast(text[i]); - - if ((byte & 0x80) == 0) { - char_len = 1; - } else if ((byte & 0xE0) == 0xC0) { - char_len = 2; - } else if ((byte & 0xF0) == 0xE0) { - char_len = 3; - } else if ((byte & 0xF8) == 0xF0) { - char_len = 4; - } - - if (i + char_len <= text.length()) { - chars.push_back(text.substr(i, char_len)); - } - i += char_len; - } - - return chars; -} - - -std::pair BPETokenizer::find_best_merge_fast(const std::vector& tokens) const { - int best_pos = -1; - uint32_t best_priority = UINT32_MAX; - - for (size_t i = 0; i < tokens.size() - 1; ++i) { - std::string key = tokens[i] + "\x00" + tokens[i + 1]; - auto it = merge_map_.find(key); - if (it != merge_map_.end()) { - if (it->second < best_priority) { - best_priority = it->second; - best_pos = static_cast(i); - } - } - } - - return {best_pos, best_priority}; -} - -std::vector BPETokenizer::apply_bpe(const std::vector& tokens) const { - if (tokens.size() <= 1) return tokens; - - std::vector current_tokens = tokens; - - - while (true) { - auto [merge_pos, priority] = find_best_merge_fast(current_tokens); - if (merge_pos == -1) break; - - - std::vector new_tokens; - new_tokens.reserve(current_tokens.size() - 1); - - for (int i = 0; i < static_cast(current_tokens.size()); ++i) { - if (i == merge_pos) { - std::string merged = current_tokens[i] + current_tokens[i + 1]; - new_tokens.push_back(merged); - i++; - } else { - new_tokens.push_back(current_tokens[i]); - } - } - current_tokens = std::move(new_tokens); - } - - return current_tokens; -} - -std::vector BPETokenizer::encode(const std::string& text) const { - if (text.empty()) return {}; - - - auto text_segments = split_with_special_tokens(text); - - - std::vector token_ids; - - for (const auto& segment : text_segments) { - auto special_it = special_tokens_.find(segment); - if (special_it != special_tokens_.end()) { - token_ids.push_back(special_it->second); - } else { - std::string normalized_segment = segment; - std::vector chars; - if (runtime_config_.normalizer == TokenizerRuntimeConfig::Normalizer::METASPACE) { - size_t pos = 0; - while ((pos = normalized_segment.find(' ', pos)) != std::string::npos) { - normalized_segment.replace(pos, 1, kMetaspace); - pos += 3; - } - chars = utf8_split(normalized_segment); - } else { - chars = byte_level_split(segment); - } - auto bpe_tokens = apply_bpe(chars); - - for (const auto& token : bpe_tokens) { - auto it = token_to_id_.find(token); - if (it != token_to_id_.end()) { - token_ids.push_back(it->second); - } else { - if (!runtime_config_.byte_fallback) { - token_ids.push_back(unk_token_id_); - continue; - } - - std::vector fallback_ids; - fallback_ids.reserve(token.size()); - for (unsigned char byte : token) { - char fallback_token[7]; - std::snprintf(fallback_token, sizeof(fallback_token), "<0x%02X>", byte); - auto fallback_it = token_to_id_.find(fallback_token); - if (fallback_it == token_to_id_.end()) { - fallback_ids.clear(); - break; - } - fallback_ids.push_back(fallback_it->second); - } - - if (fallback_ids.empty()) { - token_ids.push_back(unk_token_id_); - } else { - token_ids.insert(token_ids.end(), fallback_ids.begin(), fallback_ids.end()); - } - } - } - } - } - - return token_ids; -} - -std::string BPETokenizer::decode(const std::vector& tokens) const { - if (runtime_config_.decoder == TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE) { - std::string result; - result.reserve(tokens.size() * 4); - - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - result += id_to_token_[token_id]; - } - - size_t pos = 0; - while ((pos = result.find(kMetaspace, pos)) != std::string::npos) { - result.replace(pos, 3, " "); - pos += 1; - } - return result; - } - - std::string unicode_result; - unicode_result.reserve(tokens.size() * 4); - - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - const std::string& tok = id_to_token_[token_id]; - - size_t pos = 0; - while (pos < tok.size()) { - if (pos + 3 <= tok.size() && - (unsigned char)tok[pos] == 0xE2 && - (unsigned char)tok[pos+1] == 0x96 && - (unsigned char)tok[pos+2] == 0x81) { - unicode_result.push_back(' '); - pos += 3; - } else { - unicode_result.push_back(tok[pos++]); - } - } - } - - return unicode_to_bytes(unicode_result); -} - - -} -} diff --git a/cactus/engine/engine_cache.cpp b/cactus/engine/engine_cache.cpp deleted file mode 100644 index 77a7381b4..000000000 --- a/cactus/engine/engine_cache.cpp +++ /dev/null @@ -1,691 +0,0 @@ -#include "engine.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -void KVCache::init(size_t layers, size_t max_seq, const std::vector& layer_dims, const std::vector& layer_kv_heads, Precision model_precision) { - num_layers = layers; - max_seq_len = max_seq; - precision = model_precision; - element_size = PrecisionTraits::size_of(precision); - - layer_caches.resize(num_layers); - for (size_t i = 0; i < num_layers; i++) { - layer_caches[i].head_dim = layer_dims[i]; - layer_caches[i].kv_heads = layer_kv_heads[i]; - } - - current_seq_len = 0; - total_seq_len = 0; -} - -void KVCache::set_window_size(size_t window, size_t sink) { - window_size = window; - sink_size = sink; - - if (window_size > 0) { - for (size_t layer_idx = 0; layer_idx < num_layers; layer_idx++) { - size_t head_dim = get_layer_head_dim(layer_idx); - if (head_dim == 0) continue; - size_t layer_kv = get_layer_kv_heads(layer_idx); - size_t cache_bytes = window_size * layer_kv * head_dim * element_size; - size_t num_groups = (head_dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t num_scales = window_size * layer_kv * num_groups; - auto& cache = layer_caches[layer_idx]; - - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - std::memset(cache.keys.data(), 0, cache_bytes); - std::memset(cache.values.data(), 0, cache_bytes); - - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - std::fill(cache.key_scales.begin(), cache.key_scales.end(), 1.0f); - std::fill(cache.value_scales.begin(), cache.value_scales.end(), 1.0f); - } - } -} - -void KVCache::reset() { - current_seq_len = 0; - total_seq_len = 0; -} - -void* KVCache::get_key_ptr(size_t layer) { - if (current_seq_len == 0 || layer >= num_layers) return nullptr; - return layer_caches[layer].keys.data(); -} - -void* KVCache::get_value_ptr(size_t layer) { - if (current_seq_len == 0 || layer >= num_layers) return nullptr; - return layer_caches[layer].values.data(); -} - -KVCache::CircularView KVCache::get_key_view(size_t layer) { - CircularView view; - if (layer >= num_layers || current_seq_len == 0) { - view.ptr1 = nullptr; - view.ptr2 = nullptr; - view.len1 = 0; - view.len2 = 0; - view.total_len = 0; - return view; - } - - view.ptr1 = layer_caches[layer].keys.data(); - view.ptr2 = nullptr; - view.len1 = current_seq_len; - view.len2 = 0; - view.total_len = current_seq_len; - return view; -} - -KVCache::CircularView KVCache::get_value_view(size_t layer) { - CircularView view; - if (layer >= num_layers || current_seq_len == 0) { - view.ptr1 = nullptr; - view.ptr2 = nullptr; - view.len1 = 0; - view.len2 = 0; - view.total_len = 0; - return view; - } - - view.ptr1 = layer_caches[layer].values.data(); - view.ptr2 = nullptr; - view.len1 = current_seq_len; - view.len2 = 0; - view.total_len = current_seq_len; - return view; -} - -void KVCache::update_from_graph(CactusGraph* gb, const std::vector& k_nodes, - const std::vector& v_nodes, size_t seq_len, - size_t layers) { - size_t old_seq_len = current_seq_len; - size_t new_total_len = old_seq_len + seq_len; - - total_seq_len += seq_len; - - size_t effective_seq_len; - bool use_sliding_window = (window_size > 0 && new_total_len > window_size); - - if (use_sliding_window) { - effective_seq_len = window_size; - } else { - effective_seq_len = new_total_len; - } - - bool any_layer_updated = false; - - for (size_t layer_idx = 0; layer_idx < layers; layer_idx++) { - if (k_nodes[layer_idx] == 0 || v_nodes[layer_idx] == 0) - continue; - - auto& cache = layer_caches[layer_idx]; - - void* k_output = gb->get_output(k_nodes[layer_idx]); - void* v_output = gb->get_output(v_nodes[layer_idx]); - - if (k_output && v_output) { - const auto& k_buffer = gb->get_output_buffer(k_nodes[layer_idx]); - const auto& v_buffer = gb->get_output_buffer(v_nodes[layer_idx]); - - size_t dim = get_layer_head_dim(layer_idx); - size_t kv_heads = get_layer_kv_heads(layer_idx); - size_t elements_per_token = kv_heads * dim; - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = kv_heads * num_groups; - size_t bytes_per_token = elements_per_token * element_size; - - size_t expected_elements = new_total_len * elements_per_token; - - if (k_buffer.total_size == expected_elements && v_buffer.total_size == expected_elements) { - any_layer_updated = true; - - if (!use_sliding_window) { - size_t total_bytes = new_total_len * bytes_per_token; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - if (precision == Precision::INT8) { - size_t num_scales = new_total_len * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()), - cache.key_scales.data(), - new_total_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()), - cache.value_scales.data(), - new_total_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data(), k_output, total_bytes); - std::memcpy(cache.values.data(), v_output, total_bytes); - } - } else { - size_t cache_bytes = window_size * bytes_per_token; - size_t remaining_window = window_size - sink_size; - size_t skip_tokens = new_total_len - window_size; - - bool first_slide = (cache.keys.size() != cache_bytes); - - if (first_slide) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - } - - if (precision == Precision::INT8) { - size_t num_scales = window_size * scales_per_token; - if (first_slide) { - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - } - - const __fp16* k_fp16 = static_cast(k_output); - const __fp16* v_fp16 = static_cast(v_output); - - if (first_slide) { - cactus_quantize_kv_fp16_to_int8( - k_fp16, - reinterpret_cast(cache.keys.data()), - cache.key_scales.data(), - sink_size, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_fp16, - reinterpret_cast(cache.values.data()), - cache.value_scales.data(), - sink_size, kv_heads, dim); - } - - size_t src_offset = (sink_size + skip_tokens) * elements_per_token; - size_t dst_offset = sink_size * elements_per_token; - size_t scale_dst_offset = sink_size * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_fp16 + src_offset, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_dst_offset, - remaining_window, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_fp16 + src_offset, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_dst_offset, - remaining_window, kv_heads, dim); - } else { - if (first_slide) { - size_t sink_bytes = sink_size * bytes_per_token; - std::memcpy(cache.keys.data(), k_output, sink_bytes); - std::memcpy(cache.values.data(), v_output, sink_bytes); - } - - const uint8_t* k_src = static_cast(k_output) + - (sink_size + skip_tokens) * bytes_per_token; - const uint8_t* v_src = static_cast(v_output) + - (sink_size + skip_tokens) * bytes_per_token; - size_t sink_bytes = sink_size * bytes_per_token; - size_t recent_bytes = remaining_window * bytes_per_token; - - std::memcpy(cache.keys.data() + sink_bytes, k_src, recent_bytes); - std::memcpy(cache.values.data() + sink_bytes, v_src, recent_bytes); - } - } - } - else if (seq_len * elements_per_token == k_buffer.total_size && - seq_len * elements_per_token == v_buffer.total_size) { - any_layer_updated = true; - - if (!use_sliding_window) { - size_t old_bytes = old_seq_len * bytes_per_token; - size_t new_bytes = seq_len * bytes_per_token; - size_t total_bytes = old_bytes + new_bytes; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - if (precision == Precision::INT8) { - size_t num_scales = (old_seq_len + seq_len) * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - size_t dst_offset = old_seq_len * elements_per_token; - size_t scale_offset = old_seq_len * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - seq_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - seq_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data() + old_bytes, k_output, new_bytes); - std::memcpy(cache.values.data() + old_bytes, v_output, new_bytes); - } - } else { - size_t cache_bytes = window_size * bytes_per_token; - - if (cache.keys.size() != cache_bytes) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - } - - size_t tokens_to_shift = window_size - sink_size - seq_len; - size_t sink_bytes = sink_size * bytes_per_token; - - if (tokens_to_shift > 0 && old_seq_len > sink_size) { - size_t shift_src = old_seq_len - tokens_to_shift; - if (shift_src > sink_size) { - size_t shift_bytes = tokens_to_shift * bytes_per_token; - - std::memmove(cache.keys.data() + sink_bytes, - cache.keys.data() + shift_src * bytes_per_token, - shift_bytes); - std::memmove(cache.values.data() + sink_bytes, - cache.values.data() + shift_src * bytes_per_token, - shift_bytes); - - if (precision == Precision::INT8) { - size_t sink_scale_offset = sink_size * scales_per_token; - size_t shift_src_scale_offset = shift_src * scales_per_token; - size_t shift_scale_count = tokens_to_shift * scales_per_token; - - std::memmove(cache.key_scales.data() + sink_scale_offset, - cache.key_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - std::memmove(cache.value_scales.data() + sink_scale_offset, - cache.value_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - } - } - } - - size_t append_token_offset = window_size - seq_len; - size_t append_bytes_offset = append_token_offset * bytes_per_token; - size_t new_bytes = seq_len * bytes_per_token; - - if (precision == Precision::INT8) { - size_t append_offset = append_token_offset * elements_per_token; - size_t scale_offset = append_token_offset * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - static_cast(k_output), - reinterpret_cast(cache.keys.data()) + append_offset, - cache.key_scales.data() + scale_offset, - seq_len, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - static_cast(v_output), - reinterpret_cast(cache.values.data()) + append_offset, - cache.value_scales.data() + scale_offset, - seq_len, kv_heads, dim); - } else { - std::memcpy(cache.keys.data() + append_bytes_offset, k_output, new_bytes); - std::memcpy(cache.values.data() + append_bytes_offset, v_output, new_bytes); - } - } - } - } - } - - if (any_layer_updated) { - current_seq_len = effective_seq_len; - } -} - -void KVCache::update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data, - size_t num_tokens, size_t kv_heads, size_t dim) { - if (layer_idx >= num_layers || !k_data || !v_data || num_tokens == 0) { - return; - } - - auto& cache = layer_caches[layer_idx]; - size_t old_seq_len = current_seq_len; - size_t new_total_len = old_seq_len + num_tokens; - size_t elements_per_token = kv_heads * dim; - size_t bytes_per_token = elements_per_token * element_size; - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = kv_heads * num_groups; - - if (layer_idx == 0) { - total_seq_len += num_tokens; - } - - bool use_sliding_window = (window_size > 0 && new_total_len > window_size); - size_t effective_seq_len = use_sliding_window ? window_size : new_total_len; - - if (!use_sliding_window) { - size_t total_bytes = new_total_len * bytes_per_token; - cache.keys.resize(total_bytes); - cache.values.resize(total_bytes); - - size_t num_scales = new_total_len * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - - size_t dst_offset = old_seq_len * elements_per_token; - size_t scale_offset = old_seq_len * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - } else { - size_t cache_bytes = window_size * bytes_per_token; - - if (cache.keys.size() != cache_bytes) { - cache.keys.resize(cache_bytes); - cache.values.resize(cache_bytes); - - size_t num_scales = window_size * scales_per_token; - cache.key_scales.resize(num_scales); - cache.value_scales.resize(num_scales); - } - - size_t remaining_window = window_size - sink_size; - - if (num_tokens >= remaining_window) { - size_t skip_tokens = num_tokens - remaining_window; - size_t dst_offset = sink_size * elements_per_token; - size_t scale_offset = sink_size * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data + skip_tokens * elements_per_token, - reinterpret_cast(cache.keys.data()) + dst_offset, - cache.key_scales.data() + scale_offset, - remaining_window, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data + skip_tokens * elements_per_token, - reinterpret_cast(cache.values.data()) + dst_offset, - cache.value_scales.data() + scale_offset, - remaining_window, kv_heads, dim); - } else { - size_t tokens_to_shift = remaining_window - num_tokens; - - if (tokens_to_shift > 0 && old_seq_len > sink_size) { - size_t shift_src = old_seq_len - tokens_to_shift; - if (shift_src > sink_size) { - size_t sink_offset = sink_size * elements_per_token; - size_t shift_src_offset = shift_src * elements_per_token; - size_t shift_bytes = tokens_to_shift * bytes_per_token; - - std::memmove(cache.keys.data() + sink_offset, - cache.keys.data() + shift_src_offset, - shift_bytes); - std::memmove(cache.values.data() + sink_offset, - cache.values.data() + shift_src_offset, - shift_bytes); - - size_t sink_scale_offset = sink_size * scales_per_token; - size_t shift_src_scale_offset = shift_src * scales_per_token; - size_t shift_scale_count = tokens_to_shift * scales_per_token; - - std::memmove(cache.key_scales.data() + sink_scale_offset, - cache.key_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - std::memmove(cache.value_scales.data() + sink_scale_offset, - cache.value_scales.data() + shift_src_scale_offset, - shift_scale_count * sizeof(float)); - } - } - - size_t append_token_offset = window_size - num_tokens; - size_t append_offset = append_token_offset * elements_per_token; - size_t scale_offset = append_token_offset * scales_per_token; - - cactus_quantize_kv_fp16_to_int8( - k_data, - reinterpret_cast(cache.keys.data()) + append_offset, - cache.key_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - - cactus_quantize_kv_fp16_to_int8( - v_data, - reinterpret_cast(cache.values.data()) + append_offset, - cache.value_scales.data() + scale_offset, - num_tokens, kv_heads, dim); - } - } - - if (layer_idx == num_layers - 1) { - current_seq_len = effective_seq_len; - } -} - -void KVCache::remove_token_range(size_t start, size_t count) { - if (count == 0 || start >= current_seq_len || start + count > current_seq_len) return; - - size_t tail_tokens = current_seq_len - start - count; - - auto erase_bytes = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { - std::memmove(buf.data() + offset, buf.data() + offset + remove, tail); - buf.resize(buf.size() - remove); - }; - - auto erase_floats = [](std::vector& buf, size_t offset, size_t remove, size_t tail) { - std::memmove(buf.data() + offset, buf.data() + offset + remove, tail * sizeof(float)); - buf.resize(buf.size() - remove); - }; - - for (size_t i = 0; i < layer_caches.size(); i++) { - size_t dim = get_layer_head_dim(i); - if (dim == 0) continue; - auto& layer = layer_caches[i]; - size_t num_kv_heads = get_layer_kv_heads(i); - size_t bytes_per_tok = num_kv_heads * dim * element_size; - - erase_bytes(layer.keys, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); - erase_bytes(layer.values, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok); - - if (precision == Precision::INT8 && !layer.key_scales.empty()) { - size_t scaler_per_tok = num_kv_heads * dim / KV_QUANT_GROUP_SIZE; - erase_floats(layer.key_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); - erase_floats(layer.value_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok); - } - } - - current_seq_len -= count; - total_seq_len -= count; -} - -void KVCache::compact_to_windows(const std::vector& target_windows) { - for (size_t i = 0; i < layer_caches.size(); i++) { - size_t dim = get_layer_head_dim(i); - if (dim == 0) continue; - size_t target = (i < target_windows.size()) ? target_windows[i] : 0; - if (target == 0) continue; - - auto& layer = layer_caches[i]; - size_t num_kv_heads = get_layer_kv_heads(i); - size_t layer_tokens = layer.keys.size() / (num_kv_heads * dim * element_size); - if (layer_tokens <= target) continue; - - size_t bytes_per_token = num_kv_heads * dim * element_size; - size_t keep_bytes = target * bytes_per_token; - size_t discard_bytes = (layer_tokens - target) * bytes_per_token; - - std::memmove(layer.keys.data(), layer.keys.data() + discard_bytes, keep_bytes); - layer.keys.resize(keep_bytes); - std::memmove(layer.values.data(), layer.values.data() + discard_bytes, keep_bytes); - layer.values.resize(keep_bytes); - - if (precision == Precision::INT8 && !layer.key_scales.empty()) { - size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE; - size_t scales_per_token = num_kv_heads * num_groups; - size_t keep_scales = target * scales_per_token; - size_t discard_scales = (layer_tokens - target) * scales_per_token; - - std::memmove(layer.key_scales.data(), layer.key_scales.data() + discard_scales, keep_scales * sizeof(float)); - layer.key_scales.resize(keep_scales); - std::memmove(layer.value_scales.data(), layer.value_scales.data() + discard_scales, keep_scales * sizeof(float)); - layer.value_scales.resize(keep_scales); - } - } -} - -const int8_t* KVCache::get_keys_int8(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return reinterpret_cast(layer_caches[layer].keys.data()); -} - -const int8_t* KVCache::get_values_int8(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return reinterpret_cast(layer_caches[layer].values.data()); -} - -const float* KVCache::get_key_scales(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return layer_caches[layer].key_scales.data(); -} - -const float* KVCache::get_value_scales(size_t layer) const { - if (layer >= num_layers || current_seq_len == 0) { - return nullptr; - } - return layer_caches[layer].value_scales.data(); -} - -void ConvCache::init(size_t layers, size_t hidden_dim, size_t window_len, Precision model_precision) { - num_layers = layers; - hidden_size = hidden_dim; - window_size = window_len; - precision = model_precision; - element_size = PrecisionTraits::size_of(precision); - - size_t state_bytes = window_size * hidden_size * element_size; - layer_states.resize(num_layers); - for (auto& state : layer_states) { - state.data.resize(state_bytes); - std::memset(state.data.data(), 0, state_bytes); - state.head = 0; - state.count = 0; - } -} - -ConvCache::CircularView ConvCache::get_window(size_t layer) const { - CircularView view; - if (layer >= num_layers) { - view.ptr1 = nullptr; - view.len1 = 0; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = 0; - return view; - } - - const auto& state = layer_states[layer]; - if (state.count == 0) { - view.ptr1 = nullptr; - view.len1 = 0; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = 0; - return view; - } - - size_t stride = hidden_size * element_size; - - if (state.count < window_size) { - view.ptr1 = state.data.data(); - view.len1 = state.count; - view.ptr2 = nullptr; - view.len2 = 0; - view.total_len = state.count; - return view; - } - - view.ptr1 = state.data.data(); - view.len1 = state.head; - view.ptr2 = state.data.data() + state.head * stride; - view.len2 = window_size - state.head; - view.total_len = window_size; - return view; -} - -void ConvCache::update(CactusGraph* gb, size_t layer, const size_t bx_node) { - if (layer >= num_layers || !bx_node || window_size == 0 || hidden_size == 0) { - return; - } - - auto& state = layer_states[layer]; - const void* output_ptr = gb->get_output(bx_node); - if (!output_ptr) { - return; - } - - const auto& buffer = gb->get_output_buffer(bx_node); - const size_t stride_bytes = hidden_size * element_size; - - size_t rows = 1; - if (!buffer.shape.empty()) { - if (buffer.shape.size() == 1) { - rows = 1; - } else { - rows = buffer.shape[0]; - } - } - - if (buffer.total_size > 0 && hidden_size > 0) { - size_t inferred = buffer.total_size / hidden_size; - if (inferred > 0) { - rows = inferred; - } - } - - if (rows == 0) { - return; - } - - size_t copy_rows = std::min(rows, window_size); - size_t start_row = rows > window_size ? rows - window_size : 0; - - const uint8_t* src = static_cast(output_ptr) + start_row * stride_bytes; - - for (size_t i = 0; i < copy_rows; ++i) { - std::memcpy(state.data.data() + state.head * stride_bytes, src + i * stride_bytes, stride_bytes); - state.head = (state.head + 1) % window_size; - if (state.count < window_size) { - ++state.count; - } - } -} - -void ConvCache::reset() { - for (auto& state : layer_states) { - std::fill(state.data.begin(), state.data.end(), 0); - state.head = 0; - state.count = 0; - } -} - -} -} \ No newline at end of file diff --git a/cactus/engine/engine_constraints.cpp b/cactus/engine/engine_constraints.cpp deleted file mode 100644 index cb03cef2f..000000000 --- a/cactus/engine/engine_constraints.cpp +++ /dev/null @@ -1,1120 +0,0 @@ -#include "engine.h" - -namespace cactus { -namespace engine { - -constexpr float FORCE_BIAS = 500.0f; -constexpr float BLOCK_BIAS = -500.0f; -constexpr float NEEDLE_BLOCK_BIAS = -1e9f; - -void ToolCallConstrainer::add_tokens_for_string(const std::string& str, std::unordered_set& token_set) { - if (!tokenizer_) return; - auto tokens = tokenizer_->encode(str); - for (uint32_t t : tokens) { - token_set.insert(t); - } -} - -void ToolCallConstrainer::add_tokens_for_prefix_string(const std::string& prefix, std::unordered_set& token_set) { - if (!tokenizer_) return; - const uint32_t vocab = tokenizer_->get_vocab_size(); - for (uint32_t t = 0; t < vocab; t++) { - if (tokenizer_->decode({t}).rfind(prefix, 0) == 0) { - token_set.insert(t); - } - } -} - -void ToolCallConstrainer::tokenize_function_names(bool quote_names) { - all_func_name_tokens_.clear(); - func_name_sequences_.clear(); - - for (const auto& name : function_names_) { - std::string name_to_encode = quote_names ? ("\"" + name + "\"") : name; - auto tokens = tokenizer_->encode(name_to_encode); - func_name_sequences_[name] = tokens; - for (uint32_t t : tokens) { - all_func_name_tokens_.insert(t); - } - if (quote_names) { - auto unquoted_tokens = tokenizer_->encode(name); - for (uint32_t t : unquoted_tokens) { - all_func_name_tokens_.insert(t); - } - } - } -} - -void ToolCallConstrainer::init_common_tokens() { - backtick_tokens_.clear(); - add_tokens_for_string("`", backtick_tokens_); - add_tokens_for_string("``", backtick_tokens_); - add_tokens_for_string("```", backtick_tokens_); - add_tokens_for_string("````", backtick_tokens_); - add_tokens_for_string("```json", backtick_tokens_); - add_tokens_for_string("```JSON", backtick_tokens_); - add_tokens_for_string("``` json", backtick_tokens_); - add_tokens_for_string("```\n", backtick_tokens_); - add_tokens_for_string("` ", backtick_tokens_); -} - -void ToolCallConstrainer::needle_insert_word(NeedleTrieNode* root, const std::string& word) { - if (!root) return; - - NeedleTrieNode* node = root; - for (char ch : word) { - auto& child = node->children[ch]; - if (!child) { - child = std::make_unique(); - } - node = child.get(); - } - node->is_terminal = true; -} - -const ToolCallConstrainer::NeedleTrieNode* ToolCallConstrainer::needle_get_trie_node( - const NeedleTrieNode* root, - const std::string& prefix -) const { - const NeedleTrieNode* node = root; - for (char ch : prefix) { - if (!node) return nullptr; - auto it = node->children.find(ch); - if (it == node->children.end()) { - return nullptr; - } - node = it->second.get(); - } - return node; -} - -bool ToolCallConstrainer::needle_check_token_valid(const std::string& token_text, - const NeedleTrieNode* trie_node) const { - const NeedleTrieNode* node = trie_node; - if (!node) return false; - - for (char ch : token_text) { - if (ch == '"') { - return node->is_terminal; - } - auto it = node->children.find(ch); - if (it == node->children.end()) { - return false; - } - node = it->second.get(); - } - return true; -} - -void ToolCallConstrainer::reset_needle_constraints() { - needle_json_state_ = NeedleJsonState::FREE; - needle_buffer_.clear(); - needle_constrained_buf_.clear(); - needle_current_function_.clear(); - needle_in_arguments_ = false; - needle_arguments_depth_ = 0; - needle_nesting_depth_ = 0; - needle_in_string_value_ = false; - needle_prev_char_escape_ = false; -} - -bool ToolCallConstrainer::needle_at_arg_key_start() const { - if (needle_buffer_.size() < 2) return false; - return needle_buffer_.compare(needle_buffer_.size() - 2, 2, "{\"") == 0 || - needle_buffer_.compare(needle_buffer_.size() - 2, 2, ",\"") == 0; -} - -bool ToolCallConstrainer::needle_is_value_string_start() const { - if (needle_buffer_.empty()) return false; - for (size_t i = needle_buffer_.size() - 1; i-- > 0;) { - char ch = needle_buffer_[i]; - if (ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r') { - continue; - } - return ch == ':'; - } - return false; -} - -void ToolCallConstrainer::feed_needle_char(char ch) { - if (needle_json_state_ == NeedleJsonState::IN_NAME || - needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { - if (ch == '"') { - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - needle_current_function_ = needle_constrained_buf_; - } - needle_constrained_buf_.clear(); - needle_json_state_ = NeedleJsonState::FREE; - } else { - needle_constrained_buf_.push_back(ch); - } - needle_buffer_.push_back(ch); - return; - } - - needle_buffer_.push_back(ch); - - if (needle_in_string_value_) { - if (needle_prev_char_escape_) { - needle_prev_char_escape_ = false; - return; - } - if (ch == '\\') { - needle_prev_char_escape_ = true; - return; - } - if (ch == '"') { - needle_in_string_value_ = false; - } - return; - } - - if (ch == '{' || ch == '[') { - needle_nesting_depth_++; - } else if (ch == '}' || ch == ']') { - needle_nesting_depth_ = std::max(0, needle_nesting_depth_ - 1); - if (ch == '}' && needle_in_arguments_ && needle_nesting_depth_ < needle_arguments_depth_) { - needle_in_arguments_ = false; - } - } - - if (needle_buffer_.size() >= 8 && - needle_buffer_.compare(needle_buffer_.size() - 8, 8, "\"name\":\"") == 0) { - needle_json_state_ = NeedleJsonState::IN_NAME; - needle_constrained_buf_.clear(); - return; - } - - if (needle_buffer_.size() >= 13 && - needle_buffer_.compare(needle_buffer_.size() - 13, 13, "\"arguments\":{") == 0) { - needle_in_arguments_ = true; - needle_arguments_depth_ = needle_nesting_depth_; - return; - } - - if (needle_in_arguments_ && - needle_nesting_depth_ == needle_arguments_depth_ && - needle_at_arg_key_start()) { - needle_json_state_ = NeedleJsonState::IN_ARG_KEY; - needle_constrained_buf_.clear(); - return; - } - - if (ch == '"' && needle_is_value_string_start()) { - needle_in_string_value_ = true; - } -} - -void ToolCallConstrainer::feed_needle_text(const std::string& text) { - for (char ch : text) { - feed_needle_char(ch); - } -} - -void ToolCallConstrainer::init_needle_constraints() { - reset_needle_constraints(); - - needle_name_trie_ = std::make_unique(); - needle_param_tries_.clear(); - for (const auto& tool : tool_specs_) { - if (!tool.name.empty()) { - needle_insert_word(needle_name_trie_.get(), tool.name); - } - - auto param_root = std::make_unique(); - for (const auto& param_name : tool.parameter_names) { - if (!param_name.empty()) { - needle_insert_word(param_root.get(), param_name); - } - } - needle_param_tries_[tool.name] = std::move(param_root); - } - - const uint32_t vocab_size = tokenizer_ ? tokenizer_->get_vocab_size() : 0; - if (needle_token_strings_.size() != vocab_size) { - needle_token_strings_.assign(vocab_size, ""); - needle_token_index_.clear(); - - const uint32_t eos = tokenizer_->get_eos_token(); - const uint32_t bos = tokenizer_->get_bos_token(); - const uint32_t unk = tokenizer_->get_unk_token(); - constexpr uint32_t pad = 0; - - for (uint32_t token_id = 0; token_id < vocab_size; ++token_id) { - if (token_id == pad || token_id == eos || token_id == bos || token_id == unk) { - continue; - } - std::string decoded = tokenizer_->decode({token_id}); - needle_token_strings_[token_id] = decoded; - if (decoded.empty()) continue; - needle_token_index_[decoded.front()].push_back(token_id); - } - } -} - -void ToolCallConstrainer::tokenize_grammar_elements() { - if (!tokenizer_) return; - - // Clear all token sets - open_brace_tokens_.clear(); - close_brace_tokens_.clear(); - colon_tokens_.clear(); - comma_tokens_.clear(); - name_key_tokens_.clear(); - args_key_tokens_.clear(); - quote_tokens_.clear(); - tool_start_tokens_.clear(); - tool_end_tokens_.clear(); - bracket_open_tokens_.clear(); - bracket_close_tokens_.clear(); - paren_open_tokens_.clear(); - paren_close_tokens_.clear(); - equals_tokens_.clear(); - - init_common_tokens(); - - if (model_type_ == Config::ModelType::LFM2) { - add_tokens_for_string("<|tool_call_start|>", tool_start_tokens_); - add_tokens_for_string("<|tool_call_end|>", tool_end_tokens_); - add_tokens_for_string("[", bracket_open_tokens_); - add_tokens_for_string("]", bracket_close_tokens_); - add_tokens_for_string("(", paren_open_tokens_); - add_tokens_for_string(")", paren_close_tokens_); - add_tokens_for_string("=", equals_tokens_); - add_tokens_for_string(",", comma_tokens_); - add_tokens_for_string("\"", quote_tokens_); - - tokenize_function_names(false); - add_tokens_for_prefix_string("(", paren_open_tokens_); - } else if (is_gemma_family()) { - gemma_call_start_tokens_.clear(); - gemma_call_end_tokens_.clear(); - gemma_call_prefix_tokens_.clear(); - escape_tokens_.clear(); - gemma_response_start_tokens_.clear(); - - add_tokens_for_string(call_start_tag_, gemma_call_start_tokens_); - add_tokens_for_string(call_end_tag_, gemma_call_end_tokens_); - if (model_type_ == Config::ModelType::GEMMA4) { - add_tokens_for_string("<|tool_response>", gemma_response_start_tokens_); - } else { - add_tokens_for_string("", gemma_response_start_tokens_); - add_tokens_for_string("", escape_tokens_); - } - add_tokens_for_string("call:", gemma_call_prefix_tokens_); - - add_tokens_for_string("{", open_brace_tokens_); - add_tokens_for_string("}", close_brace_tokens_); - add_tokens_for_string(":", colon_tokens_); - add_tokens_for_string(",", comma_tokens_); - - tokenize_function_names(false); - } else if (is_needle()) { - qwen_tool_call_start_tokens_.clear(); - add_tokens_for_string("", qwen_tool_call_start_tokens_); - } else { - qwen_tool_call_start_tokens_.clear(); - qwen_tool_call_end_tokens_.clear(); - - add_tokens_for_string("", qwen_tool_call_start_tokens_); - add_tokens_for_string("", qwen_tool_call_end_tokens_); - - add_tokens_for_string("{", open_brace_tokens_); - add_tokens_for_string("}", close_brace_tokens_); - add_tokens_for_string(":", colon_tokens_); - add_tokens_for_string(",", comma_tokens_); - add_tokens_for_string("\"", quote_tokens_); - - add_tokens_for_string("name", name_key_tokens_); - - add_tokens_for_string("arguments", args_key_tokens_); - - tokenize_function_names(true); - } -} - -void ToolCallConstrainer::init(Config::ModelType model_type, - const std::vector& tools, - Tokenizer* tokenizer) { - model_type_ = model_type; - tool_specs_ = tools; - function_names_.clear(); - function_names_.reserve(tool_specs_.size()); - for (const auto& tool : tool_specs_) { - function_names_.push_back(tool.name); - } - tokenizer_ = tokenizer; - generated_text_.clear(); - brace_depth_ = 0; - active_ = !function_names_.empty() && tokenizer != nullptr; - - if (model_type_ == Config::ModelType::LFM2) { - state_ = State::LFM_START; - lfm_current_function_.clear(); - lfm_args_buffer_.clear(); - lfm_seen_arg_keys_.clear(); - lfm_required_params_.clear(); - lfm_all_params_.clear(); - for (const auto& tool : tool_specs_) { - lfm_required_params_[tool.name] = tool.required_parameter_names; - lfm_all_params_[tool.name] = tool.parameter_names; - } - } else if (is_gemma_family()) { - state_ = State::GEMMA_START; - if (model_type_ == Config::ModelType::GEMMA4) { - call_start_tag_ = "<|tool_call>"; - call_end_tag_ = ""; - } else { - call_start_tag_ = ""; - call_end_tag_ = ""; - } - } else if (is_needle()) { - state_ = State::NEEDLE_START; - } else { - state_ = State::QWEN_START; - } - - if (!active_) { - return; - } - - tokenize_grammar_elements(); - if (is_needle()) { - init_needle_constraints(); - } - compute_bias(); -} - -void ToolCallConstrainer::update(uint32_t /*token_id*/, const std::string& decoded_text) { - if (!active_) return; - - generated_text_ += decoded_text; - - if (model_type_ == Config::ModelType::LFM2) { - switch (state_) { - case State::LFM_START: - if (generated_text_.find("<|tool_call_start|>") != std::string::npos) { - state_ = State::LFM_EXPECT_BRACKET; - generated_text_.clear(); - } - break; - - case State::LFM_EXPECT_BRACKET: - if (generated_text_.find("[") != std::string::npos) { - state_ = State::LFM_IN_FUNC_NAME; - generated_text_.clear(); - } - break; - - case State::LFM_IN_FUNC_NAME: - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { - lfm_current_function_ = name; - state_ = State::LFM_EXPECT_PAREN; - generated_text_.clear(); - break; - } - } - break; - - case State::LFM_EXPECT_PAREN: - if (generated_text_.find("(") != std::string::npos) { - state_ = State::LFM_IN_ARGUMENTS; - lfm_args_buffer_.clear(); - lfm_seen_arg_keys_.clear(); - generated_text_.clear(); - } - break; - - case State::LFM_IN_ARGUMENTS: { - lfm_args_buffer_ += decoded_text; - size_t eq_pos = 0; - while ((eq_pos = lfm_args_buffer_.find('=', eq_pos)) != std::string::npos) { - size_t key_end = eq_pos; - size_t key_start = key_end; - while (key_start > 0 && !std::isspace(static_cast(lfm_args_buffer_[key_start - 1])) && - lfm_args_buffer_[key_start - 1] != ',' && lfm_args_buffer_[key_start - 1] != '(') { - key_start--; - } - if (key_start < key_end) { - std::string key = lfm_args_buffer_.substr(key_start, key_end - key_start); - lfm_seen_arg_keys_.insert(key); - } - eq_pos++; - } - if (decoded_text.find(")") != std::string::npos) { - state_ = State::LFM_EXPECT_BRACKET_CLOSE; - generated_text_.clear(); - } else { - generated_text_.clear(); - } - break; - } - - case State::LFM_EXPECT_BRACKET_CLOSE: - if (generated_text_.find("]") != std::string::npos) { - state_ = State::LFM_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::LFM_EXPECT_END: - if (generated_text_.find("<|tool_call_end|>") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - default: - break; - } - } else if (is_gemma_family()) { - switch (state_) { - case State::GEMMA_START: - if (generated_text_.find(call_start_tag_) != std::string::npos) { - state_ = State::GEMMA_EXPECT_CALL; - generated_text_.clear(); - } - break; - - case State::GEMMA_EXPECT_CALL: - if (generated_text_.find("call:") != std::string::npos) { - state_ = State::GEMMA_IN_FUNC_NAME; - generated_text_.clear(); - } - break; - - case State::GEMMA_IN_FUNC_NAME: - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { - state_ = State::GEMMA_EXPECT_BRACE; - generated_text_.clear(); - break; - } - } - break; - - case State::GEMMA_EXPECT_BRACE: - if (generated_text_.find("{") != std::string::npos) { - state_ = State::GEMMA_IN_ARGUMENTS; - brace_depth_ = 1; - generated_text_.clear(); - } - break; - - case State::GEMMA_IN_ARGUMENTS: - generated_text_.clear(); - for (char c : decoded_text) { - if (c == '{') brace_depth_++; - else if (c == '}') { - brace_depth_--; - if (brace_depth_ == 0) { - state_ = State::GEMMA_EXPECT_END; - break; - } - } - } - break; - - case State::GEMMA_EXPECT_END: - if (generated_text_.find(call_end_tag_) != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - default: - break; - } - } else if (is_needle()) { - switch (state_) { - case State::NEEDLE_START: - if (generated_text_.find("") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - case State::DONE: - feed_needle_text(decoded_text); - generated_text_.clear(); - break; - - default: - break; - } - } else { - switch (state_) { - case State::QWEN_START: - if (generated_text_.find("") != std::string::npos) { - state_ = State::QWEN_EXPECT_OPEN_BRACE; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_OPEN_BRACE: { - size_t pos = generated_text_.find("{"); - if (pos != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_KEY; - generated_text_ = generated_text_.substr(pos + 1); - } - break; - } - - case State::QWEN_EXPECT_NAME_KEY: - if (generated_text_.find("name\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_COLON; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_NAME_COLON: - if (generated_text_.find(":") != std::string::npos) { - state_ = State::QWEN_EXPECT_NAME_VALUE; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_NAME_VALUE: - for (const auto& name : function_names_) { - if (generated_text_.find(name + "\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_COMMA; - generated_text_.clear(); - break; - } - } - break; - - case State::QWEN_EXPECT_COMMA: - if (generated_text_.find(",") != std::string::npos) { - state_ = State::QWEN_EXPECT_ARGS_KEY; - generated_text_.clear(); - } else if (generated_text_.find("}") != std::string::npos) { - state_ = State::QWEN_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_ARGS_KEY: - if (generated_text_.find("arguments\"") != std::string::npos) { - state_ = State::QWEN_EXPECT_ARGS_COLON; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_ARGS_COLON: - if (generated_text_.find(":") != std::string::npos) { - state_ = State::QWEN_IN_ARGUMENTS; - brace_depth_ = 0; - generated_text_.clear(); - } - break; - - case State::QWEN_IN_ARGUMENTS: - generated_text_.clear(); - for (char c : decoded_text) { - if (c == '{') brace_depth_++; - else if (c == '}') { - if (brace_depth_ > 0) { - brace_depth_--; - } else { - state_ = State::QWEN_EXPECT_CLOSE_BRACE; - generated_text_.clear(); - break; - } - } - } - break; - - case State::QWEN_EXPECT_CLOSE_BRACE: - if (generated_text_.find("}") != std::string::npos) { - state_ = State::QWEN_EXPECT_END; - generated_text_.clear(); - } - break; - - case State::QWEN_EXPECT_END: - if (generated_text_.find("") != std::string::npos) { - state_ = State::DONE; - generated_text_.clear(); - } - break; - - case State::DONE: - if (generated_text_.find("") != std::string::npos) { - state_ = State::QWEN_EXPECT_OPEN_BRACE; - generated_text_.clear(); - } - break; - - default: - break; - } - } - - compute_bias(); -} - -void ToolCallConstrainer::compute_bias() { - current_bias_.clear(); - - if (!active_) return; - - if (!is_needle()) { - for (uint32_t t : backtick_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - } - - if (model_type_ == Config::ModelType::LFM2) { - switch (state_) { - case State::LFM_START: - for (uint32_t t : tool_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_BRACKET: - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_IN_FUNC_NAME: - for (uint32_t t : all_func_name_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_PAREN: - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_IN_ARGUMENTS: { - for (uint32_t t : equals_tokens_) { - current_bias_[t] = 10.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 8.0f; - } - for (uint32_t t : quote_tokens_) { - current_bias_[t] = 5.0f; - } - for (uint32_t t : paren_close_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : tool_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - auto req_it = lfm_required_params_.find(lfm_current_function_); - if (req_it != lfm_required_params_.end()) { - bool has_all_required = true; - for (const auto& req : req_it->second) { - if (!lfm_seen_arg_keys_.count(req)) { - has_all_required = false; - break; - } - } - if (!has_all_required) { - for (uint32_t t : paren_close_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - } - } - break; - } - - case State::LFM_EXPECT_BRACKET_CLOSE: - for (uint32_t t : bracket_close_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : equals_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::LFM_EXPECT_END: - for (uint32_t t : tool_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : bracket_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : paren_open_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } else if (is_gemma_family()) { - for (uint32_t t : gemma_response_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - - switch (state_) { - case State::GEMMA_START: - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_CALL: - for (uint32_t t : gemma_call_prefix_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_IN_FUNC_NAME: - for (uint32_t t : all_func_name_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_BRACE: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_IN_ARGUMENTS: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = 10.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 8.0f; - } - for (uint32_t t : escape_tokens_) { - current_bias_[t] = 5.0f; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::GEMMA_EXPECT_END: - for (uint32_t t : gemma_call_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : gemma_call_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } else if (is_needle()) { - switch (state_) { - case State::NEEDLE_START: - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - if (tokenizer_) { - current_bias_[tokenizer_->get_eos_token()] = FORCE_BIAS; - } - break; - - default: - if (needle_json_state_ == NeedleJsonState::FREE) { - break; - } - - const NeedleTrieNode* trie_node = nullptr; - if (needle_json_state_ == NeedleJsonState::IN_NAME) { - trie_node = needle_get_trie_node(needle_name_trie_.get(), needle_constrained_buf_); - } else if (needle_json_state_ == NeedleJsonState::IN_ARG_KEY) { - auto it = needle_param_tries_.find(needle_current_function_); - if (it != needle_param_tries_.end()) { - trie_node = needle_get_trie_node(it->second.get(), needle_constrained_buf_); - } - } - - if (!trie_node) { - break; - } - - { - std::vector valid_tokens(needle_token_strings_.size(), false); - bool has_valid = false; - auto mark_valid_tokens = [&](char first_char) { - auto idx_it = needle_token_index_.find(first_char); - if (idx_it == needle_token_index_.end()) { - return; - } - for (uint32_t token_id : idx_it->second) { - if (!valid_tokens[token_id] && - needle_check_token_valid(needle_token_strings_[token_id], trie_node)) { - valid_tokens[token_id] = true; - has_valid = true; - } - } - }; - - for (const auto& [first_char, _] : trie_node->children) { - mark_valid_tokens(first_char); - } - if (trie_node->is_terminal) { - mark_valid_tokens('"'); - } - - if (!has_valid) { - break; - } - - for (size_t token_id = 0; token_id < valid_tokens.size(); ++token_id) { - if (!valid_tokens[token_id]) { - current_bias_[static_cast(token_id)] = NEEDLE_BLOCK_BIAS; - } - } - } - break; - } - } else { - switch (state_) { - case State::QWEN_START: - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_OPEN_BRACE: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_NAME_KEY: { - bool has_name = generated_text_.find("name") != std::string::npos; - bool has_quote = generated_text_.find("\"") != std::string::npos; - if (has_name) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_quote) { - for (uint32_t t : name_key_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } - } else { - for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : args_key_tokens_) { current_bias_[t] = BLOCK_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_NAME_COLON: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_EXPECT_NAME_VALUE: { - bool name_complete = false; - for (const auto& name : function_names_) { - if (generated_text_.find(name) != std::string::npos) { name_complete = true; break; } - } - bool has_open_quote = generated_text_.find("\"") != std::string::npos; - if (name_complete) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_open_quote) { - for (uint32_t t : all_func_name_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = 5.0f; } - } else { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_COMMA: - for (uint32_t t : comma_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 5.0f; - } - break; - - case State::QWEN_EXPECT_ARGS_KEY: { - bool has_args = generated_text_.find("arguments") != std::string::npos; - bool has_quote = generated_text_.find("\"") != std::string::npos; - if (has_args) { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } else if (has_quote) { - for (uint32_t t : args_key_tokens_) { current_bias_[t] = FORCE_BIAS; } - for (uint32_t t : quote_tokens_) { current_bias_[t] = BLOCK_BIAS; } - } else { - for (uint32_t t : quote_tokens_) { current_bias_[t] = FORCE_BIAS; } - } - break; - } - - case State::QWEN_EXPECT_ARGS_COLON: - for (uint32_t t : colon_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_IN_ARGUMENTS: - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = 3.0f; - } - for (uint32_t t : colon_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : comma_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : quote_tokens_) { - current_bias_[t] = 2.0f; - } - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - case State::QWEN_EXPECT_CLOSE_BRACE: - for (uint32_t t : close_brace_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - break; - - case State::QWEN_EXPECT_END: - for (uint32_t t : qwen_tool_call_end_tokens_) { - current_bias_[t] = FORCE_BIAS; - } - for (uint32_t t : open_brace_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - for (uint32_t t : qwen_tool_call_start_tokens_) { - current_bias_[t] = BLOCK_BIAS; - } - break; - - default: - break; - } - } -} - -void ToolCallConstrainer::reset() { - generated_text_.clear(); - current_bias_.clear(); - brace_depth_ = 0; - reset_needle_constraints(); - - if (model_type_ == Config::ModelType::LFM2) { - state_ = State::LFM_START; - lfm_current_function_.clear(); - lfm_args_buffer_.clear(); - lfm_seen_arg_keys_.clear(); - } else if (is_gemma_family()) { - state_ = State::GEMMA_START; - } else if (is_needle()) { - state_ = State::NEEDLE_START; - } else { - state_ = State::QWEN_START; - } - - if (active_) { - compute_bias(); - } -} - - -void Model::set_tool_constraints(const std::vector& tools) { - tool_constrainer_.init(config_.model_type, tools, tokenizer_.get()); -} - -void Model::clear_tool_constraints() { - tool_constrainer_.reset(); - tool_constrainer_.init(config_.model_type, {}, tokenizer_.get()); -} - -void Model::update_tool_constraints(uint32_t token_id) { - if (tool_constrainer_.is_active() && tokenizer_) { - std::string decoded = tokenizer_->decode({token_id}); - tool_constrainer_.update(token_id, decoded); - } -} - -} // namespace engine -} // namespace cactus diff --git a/cactus/engine/engine_image.cpp b/cactus/engine/engine_image.cpp deleted file mode 100644 index cf066e99e..000000000 --- a/cactus/engine/engine_image.cpp +++ /dev/null @@ -1,600 +0,0 @@ -#define STB_IMAGE_IMPLEMENTATION -#define STB_IMAGE_RESIZE_IMPLEMENTATION - -#define STBI_NO_BMP -#define STBI_NO_PSD -#define STBI_NO_HDR -#define STBI_NO_PIC -#define STBI_NO_PNM -#define STBI_NO_TGA - -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Siglip2Preprocessor::PreprocessedImage::~PreprocessedImage() { - pixel_values.clear(); - pixel_attention_mask.clear(); -} - -Siglip2Preprocessor::Siglip2Preprocessor(const Config& config) - : config_(config) {} - -Siglip2Preprocessor::Siglip2Preprocessor() : config_() {} - -Siglip2Preprocessor::~Siglip2Preprocessor() = default; - -std::pair Siglip2Preprocessor::compute_pixel_limits() const { - int64_t min_pixels = static_cast(config_.min_image_tokens) * - config_.patch_size * config_.patch_size * - config_.downsample_factor * config_.downsample_factor; - int64_t max_pixels = static_cast(config_.max_image_tokens) * - config_.patch_size * config_.patch_size * - config_.downsample_factor * config_.downsample_factor; - return {min_pixels, max_pixels}; -} - -int Siglip2Preprocessor::round_by_factor(int number, int factor) { - if (factor == 0) { - return number; - } - double scaled = static_cast(number) / static_cast(factor); - long long rounded = std::llround(scaled); - - return static_cast(rounded * factor); -} - -std::pair Siglip2Preprocessor::smart_resize(int height, int width) { - const int total_factor = config_.patch_size * config_.downsample_factor; - auto [min_pixels, max_pixels] = compute_pixel_limits(); - - int h_bar = std::max(total_factor, round_by_factor(height, total_factor)); - int w_bar = std::max(total_factor, round_by_factor(width, total_factor)); - - - if (static_cast(h_bar) * static_cast(w_bar) > max_pixels) { - double beta = std::sqrt((static_cast(height) * static_cast(width)) / - static_cast(max_pixels)); - h_bar = std::max(total_factor, - static_cast(std::floor(height / beta / total_factor)) * total_factor); - w_bar = std::max(total_factor, - static_cast(std::floor(width / beta / total_factor)) * total_factor); - } else if (static_cast(h_bar) * static_cast(w_bar) < min_pixels) { - double beta = std::sqrt(static_cast(min_pixels) / - (static_cast(height) * static_cast(width))); - h_bar = std::max(total_factor, - static_cast(std::ceil(height * beta / total_factor)) * total_factor); - w_bar = std::max(total_factor, - static_cast(std::ceil(width * beta / total_factor)) * total_factor); - } - - return {w_bar, h_bar}; -} - - -bool Siglip2Preprocessor::is_image_too_large(int height, int width) { - const int total_factor = config_.patch_size * config_.downsample_factor; - const int h_bar = std::max(config_.patch_size, round_by_factor(height, total_factor)); - const int w_bar = std::max(config_.patch_size, round_by_factor(width, total_factor)); - - const int64_t pixels = static_cast(h_bar) * static_cast(w_bar); - auto [min_pixels, max_pixels] = compute_pixel_limits(); - const double max_pixels_with_tolerance = static_cast(max_pixels) * config_.max_pixels_tolerance; - - bool result = static_cast(pixels) > max_pixels_with_tolerance; - - return result; -} - - -std::pair Siglip2Preprocessor::find_closest_aspect_ratio(float aspect_ratio, int width, int height) { - float best_ratio_diff = std::numeric_limits::infinity(); - std::pair best_ratio = {1, 1}; - int area = width * height; - - std::vector> target_ratios; - for (int n = config_.min_tiles; n <= config_.max_tiles; ++n) { - for (int w = 1; w <= n; ++w) { - for (int h = 1; h <= n; ++h) { - int total_tiles = w * h; - if (total_tiles >= config_.min_tiles && total_tiles <= config_.max_tiles) { - target_ratios.push_back({w, h}); - } - } - } - } - - std::sort(target_ratios.begin(), target_ratios.end()); - target_ratios.erase(std::unique(target_ratios.begin(), target_ratios.end()), target_ratios.end()); - std::sort(target_ratios.begin(), target_ratios.end(), [](const auto& a, const auto& b) { - return (a.first * a.second) < (b.first * b.second); - }); - - for (const auto& ratio : target_ratios) { - float target_aspect_ratio = static_cast(ratio.first) / ratio.second; - float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio); - - if (ratio_diff < best_ratio_diff) { - best_ratio_diff = ratio_diff; - best_ratio = ratio; - } else if (ratio_diff == best_ratio_diff) { - int target_area = config_.tile_size * config_.tile_size * ratio.first * ratio.second; - if (area > 0.5f * target_area) { - best_ratio = ratio; - } - } - } - - - return best_ratio; -} - -std::pair Siglip2Preprocessor::get_grid_layout(int height, int width) { - float aspect_ratio = (float)width / (float)height; - auto [grid_width, grid_height] = find_closest_aspect_ratio(aspect_ratio, width, height); - - int target_width = config_.tile_size * grid_width; - int target_height = config_.tile_size * grid_height; - - return {target_width, target_height}; -} - -Siglip2Preprocessor::SpatialShapeResult Siglip2Preprocessor::compute_spatial_shapes(int height, int width) { - if (height <= 0 || width <= 0) { - throw std::runtime_error("Image dimensions must be positive"); - } - - if (config_.patch_size <= 0) { - throw std::runtime_error("Patch size must be positive"); - } - - const int patch = config_.patch_size; - auto [resized_width, resized_height] = smart_resize(height, width); - const bool should_split = config_.do_image_splitting && is_image_too_large(height, width); - - - SpatialShapeResult result; - result.grid_rows = 1; - result.grid_cols = 1; - - if (should_split) { - if (config_.tile_size % patch != 0) { - throw std::runtime_error("Tile size must be divisible by patch size"); - } - - auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); - result.grid_cols = grid_target_width / config_.tile_size; - result.grid_rows = grid_target_height / config_.tile_size; - - - const int patches_per_tile_side = config_.tile_size / patch; - const auto tile_shape = std::make_pair(patches_per_tile_side, patches_per_tile_side); - - result.shapes.reserve(static_cast(result.grid_rows) * result.grid_cols + 1); - for (int row = 0; row < result.grid_rows; ++row) { - for (int col = 0; col < result.grid_cols; ++col) { - result.shapes.push_back(tile_shape); - - } - } - - if (config_.use_thumbnail && result.grid_rows * result.grid_cols != 1) { - if (resized_height % patch != 0 || resized_width % patch != 0) { - throw std::runtime_error("Resized thumbnail dimensions must be divisible by patch size"); - } - result.shapes.emplace_back(resized_height / patch, resized_width / patch); - - } - } else { - int target_width = resized_width; - int target_height = resized_height; - if (!config_.do_resize) { - target_width = width; - target_height = height; - } - - if (target_height % patch != 0 || target_width % patch != 0) { - throw std::runtime_error("Target dimensions must be divisible by patch size"); - } - - result.shapes.emplace_back(target_height / patch, target_width / patch); - - } - - return result; -} - - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_file(const std::string& image_path) { - int width, height, channels; - unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); - - if (!img_data) { - throw std::runtime_error("Failed to load image: " + image_path + " - " + std::string(stbi_failure_reason())); - } - - PreprocessedImage result; - result = preprocess_from_memory(img_data, width, height, channels); - - stbi_image_free(img_data); - return result; -} - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::preprocess_from_memory( - const unsigned char* img_data, int width, int height, int channels) { - if (!img_data) { - throw std::runtime_error("Invalid image data pointer"); - } - - const int expected_channels = 3; - std::vector rgb_data; - const unsigned char* source_data = img_data; - int source_channels = channels; - - if (config_.do_convert_rgb && channels != expected_channels) { - rgb_data = convert_to_rgb(img_data, width, height, channels); - source_data = rgb_data.data(); - source_channels = expected_channels; - - } - - if (source_channels != expected_channels) { - throw std::runtime_error("Image must have 3 channels (RGB)"); - } - - const int patch = config_.patch_size; - const int downsample = config_.downsample_factor; - const int patch_dim = patch * patch * expected_channels; - - if (patch <= 0) { - throw std::runtime_error("Patch size must be positive"); - } - if (config_.tile_size % patch != 0) { - throw std::runtime_error("Tile size must be divisible by patch size"); - } - - auto [resized_width, resized_height] = smart_resize(height, width); - - - const int patches_per_tile_side = config_.tile_size / patch; - const int tile_patch_count = patches_per_tile_side * patches_per_tile_side; - const int thumbnail_patch_cap = config_.max_image_tokens * downsample * downsample; - int max_patches_per_tile = config_.max_num_patches; - max_patches_per_tile = std::max(max_patches_per_tile, tile_patch_count); - max_patches_per_tile = std::max(max_patches_per_tile, thumbnail_patch_cap); - - const bool allow_splitting = config_.do_image_splitting; - const bool should_split = allow_splitting && is_image_too_large(height, width); - - - size_t expected_tiles = should_split ? static_cast(config_.max_tiles) : 1; - if (should_split && config_.use_thumbnail) { - expected_tiles += 1; - } - - std::vector> tile_patches; - tile_patches.reserve(expected_tiles); - std::vector> spatial_shapes; - spatial_shapes.reserve(expected_tiles); - - auto normalize_and_patchify = [&](const float* data_ptr, int img_width, int img_height) { - if (img_height % patch != 0 || img_width % patch != 0) { - throw std::runtime_error("Image dimensions must be divisible by patch size"); - } - std::vector normalized = normalize_image(data_ptr, img_width, img_height, expected_channels); - auto patches = convert_image_to_patches(normalized, img_width, img_height, expected_channels, patch); - std::vector flattened(patches.size() * patch_dim); - for (size_t idx = 0; idx < patches.size(); ++idx) { - std::copy(patches[idx].begin(), patches[idx].end(), flattened.begin() + idx * patch_dim); - } - tile_patches.push_back(std::move(flattened)); - spatial_shapes.emplace_back(img_height / patch, img_width / patch); - }; - - int grid_rows = 1; - int grid_cols = 1; - bool thumbnail_added = false; - - if (should_split) { - auto [grid_target_width, grid_target_height] = get_grid_layout(height, width); - grid_cols = grid_target_width / config_.tile_size; - grid_rows = grid_target_height / config_.tile_size; - - std::vector resized_grid = resize_image( - source_data, width, height, grid_target_width, grid_target_height, expected_channels); - - std::vector tile_buffer( - static_cast(config_.tile_size) * config_.tile_size * expected_channels); - - for (int row = 0; row < grid_rows; ++row) { - for (int col = 0; col < grid_cols; ++col) { - for (int y = 0; y < config_.tile_size; ++y) { - const float* src_row = resized_grid.data() + - ((row * config_.tile_size + y) * grid_target_width + col * config_.tile_size) * - expected_channels; - float* dst_row = tile_buffer.data() + (y * config_.tile_size) * expected_channels; - std::copy_n(src_row, - static_cast(config_.tile_size) * expected_channels, - dst_row); - } - normalize_and_patchify(tile_buffer.data(), config_.tile_size, config_.tile_size); - - } - } - - if (config_.use_thumbnail && grid_rows * grid_cols != 1) { - std::vector thumbnail_bytes = resize_image( - source_data, width, height, resized_width, resized_height, expected_channels); - normalize_and_patchify(thumbnail_bytes.data(), resized_width, resized_height); - thumbnail_added = true; - - } - } else { - int target_width = resized_width; - int target_height = resized_height; - - const bool needs_resize = config_.do_resize && (width != target_width || height != target_height); - - std::vector resized_image; - if (needs_resize) { - resized_image = resize_image(source_data, width, height, target_width, target_height, expected_channels); - normalize_and_patchify(resized_image.data(), target_width, target_height); - - } else { - resized_image.resize(static_cast(width) * height * expected_channels); - for (size_t idx = 0; idx < resized_image.size(); ++idx) { - resized_image[idx] = static_cast(source_data[idx]); - } - normalize_and_patchify(resized_image.data(), width, height); - target_width = width; - target_height = height; - resized_width = target_width; - resized_height = target_height; - - } - - grid_rows = 1; - grid_cols = 1; - } - - PreprocessedImage result = pad_patches(tile_patches, spatial_shapes, patch_dim, max_patches_per_tile); - - - result.image_rows = grid_rows; - result.image_cols = grid_cols; - result.image_width = resized_width; - result.image_height = resized_height; - - auto compute_tokens = [&](int patches_h, int patches_w) -> int { - int tokens_h = (patches_h + downsample - 1) / downsample; - int tokens_w = (patches_w + downsample - 1) / downsample; - return tokens_h * tokens_w; - }; - - result.tokens_per_tile = spatial_shapes.empty() - ? 0 - : compute_tokens(spatial_shapes.front().first, spatial_shapes.front().second); - - - if (thumbnail_added && !spatial_shapes.empty()) { - const auto& thumb_shape = spatial_shapes.back(); - result.thumbnail_tokens = compute_tokens(thumb_shape.first, thumb_shape.second); - - } else { - result.thumbnail_tokens = 0; - } - - return result; -} - -std::vector Siglip2Preprocessor::convert_to_rgb( - const unsigned char* img_data, int width, int height, int channels) { - - std::vector rgb_data(width * height * 3); - - if (channels == 1) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i]; - rgb_data[i * 3 + 1] = img_data[i]; - rgb_data[i * 3 + 2] = img_data[i]; - } - } else if (channels == 4) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i * 4 + 0]; - rgb_data[i * 3 + 1] = img_data[i * 4 + 1]; - rgb_data[i * 3 + 2] = img_data[i * 4 + 2]; - } - } else if (channels == 2) { - for (int i = 0; i < width * height; ++i) { - rgb_data[i * 3 + 0] = img_data[i * 2 + 0]; - rgb_data[i * 3 + 1] = img_data[i * 2 + 0]; - rgb_data[i * 3 + 2] = img_data[i * 2 + 0]; - } - } else { - throw std::runtime_error("Unsupported number of channels: " + std::to_string(channels)); - } - - return rgb_data; -} - -std::vector Siglip2Preprocessor::resize_image( - const unsigned char* img_data, int src_width, int src_height, - int dst_width, int dst_height, int channels) { - - const size_t src_elements = static_cast(src_width) * src_height * channels; - std::vector src_float(src_elements); - for (size_t idx = 0; idx < src_elements; ++idx) { - src_float[idx] = static_cast(img_data[idx]); - } - - - std::vector resized_data(static_cast(dst_width) * dst_height * channels); - - stbir_pixel_layout layout = (channels == 1) ? STBIR_1CHANNEL : - (channels == 3) ? STBIR_RGB : STBIR_RGBA; - - float* result = stbir_resize_float_linear( - src_float.data(), src_width, src_height, 0, - resized_data.data(), dst_width, dst_height, 0, - layout - ); - - if (!result) { - throw std::runtime_error("Failed to resize image"); - } - - - return resized_data; -} - -std::vector Siglip2Preprocessor::normalize_image( - const float* img_data, int width, int height, int channels) { - - size_t total_pixels = width * height * channels; - std::vector normalized(total_pixels); - - for (size_t i = 0; i < static_cast(width * height); ++i) { - for (int c = 0; c < channels; ++c) { - size_t idx = i * channels + c; - float pixel = img_data[idx]; - - if (config_.do_rescale) { - pixel *= config_.rescale_factor; - } - - if (config_.do_normalize) { - pixel = (pixel - config_.image_mean[c]) / config_.image_std[c]; - } - - normalized[idx] = pixel; - } - } - - - return normalized; -} - -std::vector> Siglip2Preprocessor::convert_image_to_patches( - const std::vector& image, int width, int height, int channels, int patch_size) { - - int num_patches_height = height / patch_size; - int num_patches_width = width / patch_size; - int num_patches = num_patches_height * num_patches_width; - int patch_elements = patch_size * patch_size * channels; - - std::vector> patches(num_patches, std::vector(patch_elements)); - - for (int ph = 0; ph < num_patches_height; ++ph) { - for (int pw = 0; pw < num_patches_width; ++pw) { - int patch_idx = ph * num_patches_width + pw; - - for (int y = 0; y < patch_size; ++y) { - for (int x = 0; x < patch_size; ++x) { - int img_y = ph * patch_size + y; - int img_x = pw * patch_size + x; - int img_idx = (img_y * width + img_x) * channels; - int patch_offset = (y * patch_size + x) * channels; - - for (int c = 0; c < channels; ++c) { - patches[patch_idx][patch_offset + c] = image[img_idx + c]; - } - } - } - } - } - - return patches; -} - -Siglip2Preprocessor::PreprocessedImage Siglip2Preprocessor::pad_patches( - const std::vector>& tile_patches, - const std::vector>& spatial_shapes, - int patch_dim, - int max_patches_per_tile) { - - if (tile_patches.size() != spatial_shapes.size()) { - throw std::runtime_error("Mismatch between tile data and spatial shapes"); - } - - PreprocessedImage result; - - const int num_tiles = static_cast(tile_patches.size()); - result.num_tiles = num_tiles; - result.patch_dim = patch_dim; - result.max_patches_per_tile = max_patches_per_tile; - result.spatial_shapes = spatial_shapes; - result.actual_num_patches = 0; - - const size_t total_values = static_cast(num_tiles) * max_patches_per_tile * patch_dim; - result.pixel_values.assign(total_values, 0.0f); - result.pixel_attention_mask.assign(static_cast(num_tiles) * max_patches_per_tile, 0); - - - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - const auto& [patches_h, patches_w] = spatial_shapes[tile_idx]; - const int actual_patches = patches_h * patches_w; - - if (actual_patches > max_patches_per_tile) { - throw std::runtime_error("Actual patches exceed max_patches_per_tile"); - } - - const auto& flattened = tile_patches[tile_idx]; - const size_t expected_size = static_cast(actual_patches) * patch_dim; - if (flattened.size() != expected_size) { - throw std::runtime_error("Tile patch data has unexpected size"); - } - - float* destination = result.pixel_values.data() + - static_cast(tile_idx) * max_patches_per_tile * patch_dim; - std::memcpy(destination, flattened.data(), expected_size * sizeof(float)); - - - int mask_offset = tile_idx * max_patches_per_tile; - for (int p = 0; p < actual_patches; ++p) { - result.pixel_attention_mask[mask_offset + p] = 1; - } - - result.actual_num_patches += actual_patches; - } - - if (!spatial_shapes.empty()) { - result.num_patches_height = spatial_shapes.front().first; - result.num_patches_width = spatial_shapes.front().second; - } else { - result.num_patches_height = 0; - result.num_patches_width = 0; - } - - result.pixel_values_shape = { - static_cast(num_tiles), - static_cast(max_patches_per_tile), - static_cast(patch_dim) - }; - - - result.pixel_attention_mask_shape = { - static_cast(num_tiles), - static_cast(max_patches_per_tile) - }; - - result.spatial_shapes_shape = { - static_cast(num_tiles), - static_cast(2) - }; - - - return result; -} - -} // namespace engine -} // namespace cactus - - diff --git a/cactus/engine/engine_index.cpp b/cactus/engine/engine_index.cpp deleted file mode 100644 index 13d08920a..000000000 --- a/cactus/engine/engine_index.cpp +++ /dev/null @@ -1,780 +0,0 @@ -#include "engine.h" -#include "kernel/kernel.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { -namespace index { - - __fp16 dot_product(const __fp16 *a, const __fp16 *b, size_t dim) { - __fp16 result; - cactus_matmul_f16(a, b, &result, 1, dim, 1); - return result; - } - - void normalize(__fp16 *v, size_t dim) { - __fp16 x = dot_product(v, v, dim); - if (x < 1e-6f) { - throw std::runtime_error("Cannot normalize zero vector"); - } - cactus_scalar_op_f16(&x, &x, 1, 0, ScalarOpType::SQRT); - cactus_scalar_op_f16(v, v, dim, x, ScalarOpType::DIVIDE); - } - - Index::Index(const std::string& index_path, const std::string& data_path, size_t embedding_dim): - index_path_(index_path), data_path_(data_path), embedding_dim_(embedding_dim), - index_entry_size_(IndexEntry::size(embedding_dim)), num_documents_(0), - index_fd_(-1), data_fd_(-1), - mapped_index_(nullptr), mapped_data_(nullptr) { - - bool index_exists = (access(index_path.c_str(), F_OK) == 0); - bool data_exists = (access(data_path.c_str(), F_OK) == 0); - - if (index_exists != data_exists) { - throw std::runtime_error("Index and data files must both exist or both not exist"); - } - - index_fd_ = open(index_path.c_str(), O_RDWR | O_CREAT, 0644); - if (index_fd_ < 0) { - throw std::runtime_error("Cannot open index file: " + index_path); - } - - data_fd_ = open(data_path.c_str(), O_RDWR | O_CREAT, 0644); - if (data_fd_ < 0) { - close(index_fd_); - throw std::runtime_error("Cannot open data file: " + data_path); - } - - auto cleanup_and_throw = [this](const std::string& msg) { - close(index_fd_); - close(data_fd_); - throw std::runtime_error(msg); - }; - - struct stat index_st, data_st; - if (fstat(index_fd_, &index_st)) { - cleanup_and_throw("Cannot get index file size: " + index_path); - } - index_file_size_ = index_st.st_size; - - if (fstat(data_fd_, &data_st)) { - cleanup_and_throw("Cannot get data file size: " + data_path); - } - data_file_size_ = data_st.st_size; - - if (!index_exists) { - index_file_size_ = sizeof(IndexHeader); - data_file_size_ = sizeof(DataHeader); - - if (ftruncate(index_fd_, index_file_size_) != 0) { - cleanup_and_throw("Failed to resize index file"); - } - - if (ftruncate(data_fd_, data_file_size_) != 0) { - cleanup_and_throw("Failed to resize data file"); - } - } - - mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); - if (mapped_index_ == MAP_FAILED) { - cleanup_and_throw("Cannot map file: " + index_path); - } - - mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, index_file_size_); - close(index_fd_); - close(data_fd_); - throw std::runtime_error("Cannot map file: " + data_path); - } - - if (!index_exists) { - IndexHeader index_header = { - MAGIC, - VERSION, - static_cast(embedding_dim), - 0 - }; - memcpy(mapped_index_, &index_header, sizeof(IndexHeader)); - - DataHeader data_header = { MAGIC, VERSION }; - memcpy(mapped_data_, &data_header, sizeof(DataHeader)); - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - cleanup_and_throw("Failed to sync index file"); - } - - if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { - cleanup_and_throw("Failed to sync data file"); - } - } - - parse_index_header(); - parse_data_header(); - - build_doc_id_map(); - } - - Index::~Index() { - if (mapped_index_ != nullptr && mapped_index_ != MAP_FAILED) { - madvise(mapped_index_, index_file_size_, MADV_DONTNEED); - munmap(mapped_index_, index_file_size_); - } - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - madvise(mapped_data_, data_file_size_, MADV_DONTNEED); - munmap(mapped_data_, data_file_size_); - } - if (index_fd_ != -1) { - close(index_fd_); - } - if (data_fd_ != -1) { - close(data_fd_); - } - } - - void Index::add_documents(const std::vector& documents) { - validate_documents(documents); - - size_t added_data_size = 0; - for (const auto& doc : documents) { - added_data_size += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); - } - - size_t new_index_size = index_file_size_ + documents.size() * index_entry_size_; - size_t new_data_size = data_file_size_ + added_data_size; - - if (ftruncate(index_fd_, new_index_size) != 0) { - throw std::runtime_error("Failed to resize index file"); - } - - if (ftruncate(data_fd_, new_data_size) != 0) { - throw std::runtime_error("Failed to resize data file"); - } - - munmap(mapped_index_, index_file_size_); - munmap(mapped_data_, data_file_size_); - - mapped_index_ = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, index_fd_, 0); - if (mapped_index_ == MAP_FAILED) { - throw std::runtime_error("Failed to remap index file"); - } - - mapped_data_ = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, data_fd_, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, new_index_size); - throw std::runtime_error("Failed to remap data file"); - } - - size_t old_index_size = index_file_size_; - size_t old_data_size = data_file_size_; - - index_file_size_ = new_index_size; - data_file_size_ = new_data_size; - - char* index_write_pos = static_cast(mapped_index_) + old_index_size; - char* data_write_pos = static_cast(mapped_data_) + old_data_size; - - uint64_t data_offset = old_data_size; - doc_id_map_.reserve(doc_id_map_.size() + documents.size()); - - for (size_t i = 0; i < documents.size(); ++i) { - const auto& doc = documents[i]; - - DataEntry data_entry{ - static_cast(doc.content.size()), - static_cast(doc.metadata.size()) - }; - - memcpy(data_write_pos, &data_entry, sizeof(DataEntry)); - data_write_pos += sizeof(DataEntry); - memcpy(data_write_pos, doc.content.data(), doc.content.size()); - data_write_pos += doc.content.size(); - memcpy(data_write_pos, doc.metadata.data(), doc.metadata.size()); - data_write_pos += doc.metadata.size(); - - IndexEntry index_entry{doc.id, data_offset, 0}; - memcpy(index_write_pos, &index_entry, sizeof(IndexEntry)); - index_write_pos += sizeof(IndexEntry); - - std::vector<__fp16> normalized_embedding(embedding_dim_); - cactus_fp32_to_fp16(doc.embedding.data(), normalized_embedding.data(), embedding_dim_); - normalize(normalized_embedding.data(), embedding_dim_); - - size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); - memcpy(index_write_pos, normalized_embedding.data(), embedding_bytes); - index_write_pos += embedding_bytes; - - doc_id_map_[doc.id] = num_documents_ + static_cast(i); - data_offset += sizeof(DataEntry) + doc.content.size() + doc.metadata.size(); - } - - num_documents_ += static_cast(documents.size()); - IndexHeader* header = reinterpret_cast(mapped_index_); - header->num_documents = num_documents_; - - if (msync(mapped_data_, data_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync data file to disk"); - } - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync index file to disk"); - } - } - - void Index::delete_documents(const std::vector& doc_ids) { - validate_doc_ids(doc_ids); - - char* index_ptr = static_cast(mapped_index_); - char* entries = index_ptr + sizeof(IndexHeader); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - for (int doc_id : doc_ids) { - uint32_t i = doc_id_map_.at(doc_id); - IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - entry.flags |= 0x1; - doc_id_map_.erase(doc_id); - } - - if (msync(mapped_index_, index_file_size_, MS_SYNC) != 0) { - throw std::runtime_error("Failed to sync index file to disk"); - } - } - - std::vector Index::get_documents(const std::vector& doc_ids) { - validate_doc_ids(doc_ids); - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - const char* data_ptr = static_cast(mapped_data_); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - std::vector results; - results.reserve(doc_ids.size()); - - for (int doc_id : doc_ids) { - uint32_t i = doc_id_map_.at(doc_id); - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { - throw std::runtime_error("File corrupted: data entry extends beyond file size"); - } - - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - size_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { - throw std::runtime_error("File corrupted: data entry extends beyond file size"); - } - - std::vector embedding_f32(embedding_dim_); - cactus_fp16_to_fp32(entry.embedding(), embedding_f32.data(), embedding_dim_); - - results.emplace_back( - doc_id, - std::move(embedding_f32), - std::string(data_entry->content(), data_entry->content_len), - std::string(data_entry->metadata(), data_entry->metadata_len) - ); - } - - return results; - } - - std::vector> Index::query(const std::vector>& embeddings, const QueryOptions& options) { - if (embeddings.empty()) { - return {}; - } - - if (options.top_k == 0) { - return std::vector>(embeddings.size()); - } - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - std::vector> normalized_embeddings; - normalized_embeddings.reserve(embeddings.size()); - - for (const auto& embedding : embeddings) { - std::vector<__fp16> normalized_embedding(embedding_dim_); - cactus_fp32_to_fp16(embedding.data(), normalized_embedding.data(), embedding_dim_); - normalize(normalized_embedding.data(), embedding_dim_); - normalized_embeddings.emplace_back(std::move(normalized_embedding)); - } - - std::vector> all_results; - all_results.reserve(embeddings.size()); - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - - auto cmp = [](const QueryResult& a, const QueryResult& b) { - return a.score > b.score; - }; - - for (const auto& normalized_embedding : normalized_embeddings) { - std::priority_queue, decltype(cmp)> top_results(cmp); - - for (uint32_t i = 0; i < num_documents_; ++i) { - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (entry.flags & 0x1) { - continue; - } - - float score = static_cast( - dot_product(normalized_embedding.data(), entry.embedding(), embedding_dim_) - ); - - if (score < options.score_threshold) { - continue; - } - - if (top_results.size() < options.top_k) { - top_results.emplace(entry.doc_id, score); - } else if (score > top_results.top().score) { - top_results.pop(); - top_results.emplace(entry.doc_id, score); - } - } - - std::vector results; - results.reserve(top_results.size()); - - while(!top_results.empty()) { - results.emplace_back(top_results.top()); - top_results.pop(); - } - std::reverse(results.begin(), results.end()); - - all_results.emplace_back(std::move(results)); - } - - return all_results; - } - - void Index::compact() { - std::string temp_index_path = index_path_ + ".tmp"; - std::string temp_data_path = data_path_ + ".tmp"; - - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - const char* data_ptr = static_cast(mapped_data_); - - uint32_t compacted_count = static_cast(doc_id_map_.size()); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: index entry extends beyond file size"); - } - } - - off_t new_data_offset = sizeof(DataHeader); - size_t new_data_size = sizeof(DataHeader); - - for (const auto& [doc_id, index] : doc_id_map_) { - const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); - - if (static_cast(entry.data_offset) + sizeof(DataEntry) > data_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); - } - - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - if (static_cast(entry.data_offset) + data_entry_size > data_file_size_) { - throw std::runtime_error("Compaction failed: File corrupted: data entry extends beyond file size"); - } - - new_data_size += data_entry_size; - } - - size_t new_index_size = sizeof(IndexHeader) + compacted_count * index_entry_size_; - - int temp_index_fd = open(temp_index_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); - if (temp_index_fd < 0) { - throw std::runtime_error("Cannot create temporary index file: " + temp_index_path); - } - - int temp_data_fd = open(temp_data_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0644); - if (temp_data_fd < 0) { - close(temp_index_fd); - unlink(temp_index_path.c_str()); - throw std::runtime_error("Cannot create temporary data file: " + temp_data_path); - } - - auto cleanup_and_throw = [&temp_index_fd, &temp_data_fd, &temp_index_path, &temp_data_path](const std::string& msg) { - close(temp_index_fd); - close(temp_data_fd); - unlink(temp_index_path.c_str()); - unlink(temp_data_path.c_str()); - throw std::runtime_error(msg); - }; - - if (ftruncate(temp_index_fd, new_index_size) != 0) { - cleanup_and_throw("Failed to resize temporary index file"); - } - - if (ftruncate(temp_data_fd, new_data_size) != 0) { - cleanup_and_throw("Failed to resize temporary data file"); - } - - void* temp_index_map = mmap(nullptr, new_index_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_index_fd, 0); - if (temp_index_map == MAP_FAILED) { - cleanup_and_throw("Cannot map temporary index file"); - } - - void* temp_data_map = mmap(nullptr, new_data_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_data_fd, 0); - if (temp_data_map == MAP_FAILED) { - munmap(temp_index_map, new_index_size); - cleanup_and_throw("Cannot map temporary data file"); - } - - IndexHeader new_header = { - MAGIC, - VERSION, - static_cast(embedding_dim_), - compacted_count - }; - - DataHeader new_data_header = { - MAGIC, - VERSION - }; - - memcpy(temp_index_map, &new_header, sizeof(IndexHeader)); - memcpy(temp_data_map, &new_data_header, sizeof(DataHeader)); - - char* temp_index_ptr = static_cast(temp_index_map); - char* temp_data_ptr = static_cast(temp_data_map); - char* index_write_pos = temp_index_ptr + sizeof(IndexHeader); - char* data_write_pos = temp_data_ptr + sizeof(DataHeader); - - std::unordered_map new_doc_id_map; - new_doc_id_map.reserve(compacted_count); - - uint32_t new_index = 0; - new_data_offset = sizeof(DataHeader); - - for (const auto& [doc_id, index] : doc_id_map_) { - const IndexEntry& entry = *reinterpret_cast(entries + index * index_entry_size_); - const DataEntry* data_entry = reinterpret_cast(data_ptr + entry.data_offset); - uint32_t data_entry_size = sizeof(DataEntry) + data_entry->content_len + data_entry->metadata_len; - - memcpy(data_write_pos, data_entry, data_entry_size); - data_write_pos += data_entry_size; - - IndexEntry new_entry = {entry.doc_id, static_cast(new_data_offset), 0}; - - memcpy(index_write_pos, &new_entry, sizeof(IndexEntry)); - index_write_pos += sizeof(IndexEntry); - - size_t embedding_bytes = embedding_dim_ * sizeof(__fp16); - memcpy(index_write_pos, entry.embedding(), embedding_bytes); - index_write_pos += embedding_bytes; - - new_doc_id_map[doc_id] = new_index; - ++new_index; - - if (static_cast(data_entry_size) > std::numeric_limits::max() - new_data_offset) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Data offset overflow during compaction"); - } - new_data_offset += data_entry_size; - } - - if (msync(temp_data_map, new_data_size, MS_SYNC) != 0) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Failed to sync temporary data file"); - } - - if (msync(temp_index_map, new_index_size, MS_SYNC) != 0) { - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - cleanup_and_throw("Failed to sync temporary index file"); - } - - munmap(temp_index_map, new_index_size); - munmap(temp_data_map, new_data_size); - - std::string backup_index = index_path_ + ".backup"; - std::string backup_data = data_path_ + ".backup"; - - if (access(backup_index.c_str(), F_OK) == 0 || access(backup_data.c_str(), F_OK) == 0) { - cleanup_and_throw("Backup files already exist, previous compaction may have failed"); - } - - close(temp_index_fd); - close(temp_data_fd); - - munmap(mapped_index_, index_file_size_); - munmap(mapped_data_, data_file_size_); - close(index_fd_); - close(data_fd_); - - if (rename(index_path_.c_str(), backup_index.c_str()) != 0) { - cleanup_and_throw("Failed to backup index file"); - } - if (rename(data_path_.c_str(), backup_data.c_str()) != 0) { - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to backup data file"); - } - - if (rename(temp_data_path.c_str(), data_path_.c_str()) != 0) { - rename(backup_data.c_str(), data_path_.c_str()); - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to rename data file"); - } - - if (rename(temp_index_path.c_str(), index_path_.c_str()) != 0) { - unlink(data_path_.c_str()); - rename(backup_data.c_str(), data_path_.c_str()); - rename(backup_index.c_str(), index_path_.c_str()); - cleanup_and_throw("Failed to rename index file"); - } - - auto sync_dir = [](const std::string& file_path) { - size_t last_slash = file_path.rfind('/'); - std::string dir_path = (last_slash != std::string::npos) ? file_path.substr(0, last_slash) : "."; - int dir_fd = open(dir_path.c_str(), O_RDONLY); - if (dir_fd < 0) { - throw std::runtime_error("Cannot open directory for fsync: " + dir_path); - } - if (fsync(dir_fd) != 0) { - close(dir_fd); - throw std::runtime_error("Failed to fsync directory: " + dir_path); - } - close(dir_fd); - }; - - sync_dir(index_path_); - sync_dir(data_path_); - - unlink(backup_index.c_str()); - unlink(backup_data.c_str()); - - int new_index_fd = open(index_path_.c_str(), O_RDWR); - if (new_index_fd < 0) { - throw std::runtime_error("Cannot open new index file: " + index_path_); - } - - int new_data_fd = open(data_path_.c_str(), O_RDWR); - if (new_data_fd < 0) { - close(new_index_fd); - throw std::runtime_error("Cannot open new data file: " + data_path_); - } - - auto cleanup_fds = [&new_index_fd, &new_data_fd](const std::string& msg) { - close(new_index_fd); - close(new_data_fd); - throw std::runtime_error(msg); - }; - - struct stat index_st, data_st; - if (fstat(new_index_fd, &index_st)) { - cleanup_fds("Cannot get index file size: " + index_path_); - } - index_file_size_ = index_st.st_size; - - if (fstat(new_data_fd, &data_st)) { - cleanup_fds("Cannot get data file size: " + data_path_); - } - data_file_size_ = data_st.st_size; - - mapped_index_ = mmap(nullptr, index_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_index_fd, 0); - if (mapped_index_ == MAP_FAILED) { - cleanup_fds("Cannot map file: " + index_path_); - } - - mapped_data_ = mmap(nullptr, data_file_size_, PROT_READ | PROT_WRITE, MAP_SHARED, new_data_fd, 0); - if (mapped_data_ == MAP_FAILED) { - munmap(mapped_index_, index_file_size_); - close(new_index_fd); - close(new_data_fd); - throw std::runtime_error("Cannot map file: " + data_path_); - } - - index_fd_ = new_index_fd; - data_fd_ = new_data_fd; - - num_documents_ = compacted_count; - doc_id_map_ = std::move(new_doc_id_map); - } - - void Index::parse_index_header() { - const char* index_ptr = static_cast(mapped_index_); - size_t offset = 0; - - if (index_file_size_ < sizeof(IndexHeader)) { - throw std::runtime_error("Index file too small: insufficient data for header"); - } - - IndexHeader header; - header.magic = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.magic); - - if (header.magic != MAGIC) { - throw std::runtime_error("Invalid index file magic number"); - } - - header.version = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.version); - - if (header.version != VERSION) { - throw std::runtime_error("Index file version mismatch"); - } - - header.embedding_dim = *reinterpret_cast(index_ptr + offset); - offset += sizeof(header.embedding_dim); - - if (static_cast(header.embedding_dim) != embedding_dim_) { - throw std::runtime_error("Embedding dimension mismatch"); - } - - num_documents_ = *reinterpret_cast(index_ptr + offset); - } - - void Index::parse_data_header() { - const char* data_ptr = static_cast(mapped_data_); - size_t offset = 0; - - if (data_file_size_ < sizeof(DataHeader)) { - throw std::runtime_error("Data file too small: insufficient data for header"); - } - - DataHeader header; - header.magic = *reinterpret_cast(data_ptr + offset); - offset += sizeof(header.magic); - - if (header.magic != MAGIC) { - throw std::runtime_error("Invalid data file magic number"); - } - - header.version = *reinterpret_cast(data_ptr + offset); - offset += sizeof(header.version); - - if (header.version != VERSION) { - throw std::runtime_error("Data file version mismatch"); - } - } - - void Index::build_doc_id_map() { - const char* index_ptr = static_cast(mapped_index_); - const char* entries = index_ptr + sizeof(IndexHeader); - - if (num_documents_ > 0) { - size_t last_entry_offset = (num_documents_ - 1) * index_entry_size_; - if (sizeof(IndexHeader) + last_entry_offset + index_entry_size_ > index_file_size_) { - throw std::runtime_error("File corrupted: index entry extends beyond file size"); - } - } - - doc_id_map_.reserve(static_cast(num_documents_)); - - for (uint32_t i = 0; i < num_documents_; ++i) { - const IndexEntry& entry = *reinterpret_cast(entries + i * index_entry_size_); - - if (entry.flags & 0x1) { - continue; - } - - doc_id_map_[entry.doc_id] = i; - } - } - - void Index::validate_documents(const std::vector& documents) { - if (documents.empty()) { - throw std::runtime_error("Documents vector is empty"); - } - - std::unordered_set seen_ids; - for (const auto& doc : documents) { - if (doc.content.size() > 65535) { - throw std::runtime_error("Document content too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); - } - - if (doc.metadata.size() > 65535) { - throw std::runtime_error("Document metadata too long (max 65535 bytes) for ID: " + std::to_string(doc.id)); - } - - if (doc.embedding.size() != embedding_dim_) { - throw std::runtime_error("Document embedding dimension mismatch for ID: " + std::to_string(doc.id)); - } - - if (seen_ids.find(doc.id) != seen_ids.end()) { - throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc.id)); - } - seen_ids.insert(doc.id); - - if (doc_id_map_.find(doc.id) != doc_id_map_.end()) { - throw std::runtime_error("Document ID already exists: " + std::to_string(doc.id)); - } - } - } - - void Index::validate_doc_ids(const std::vector& doc_ids) { - if (doc_ids.empty()) { - throw std::runtime_error("Document IDs vector is empty"); - } - - std::unordered_set seen_ids; - for (int doc_id : doc_ids) { - if (seen_ids.find(doc_id) != seen_ids.end()) { - throw std::runtime_error("Duplicate document ID in input: " + std::to_string(doc_id)); - } - seen_ids.insert(doc_id); - - if (doc_id_map_.find(doc_id) == doc_id_map_.end()) { - throw std::runtime_error("Document ID not found: " + std::to_string(doc_id)); - } - } - } - - ssize_t Index::write_full(int fd, const void* buf, size_t count) { - size_t written = 0; - const char* ptr = static_cast(buf); - while (written < count) { - ssize_t n = write(fd, ptr + written, count - written); - if (n < 0) { - if (errno == EINTR) continue; - return -1; - } - if (n == 0) return written; - written += n; - } - return written; - } - -} // namespace index -} // namespace engine -} // namespace cactus diff --git a/cactus/engine/engine_model.cpp b/cactus/engine/engine_model.cpp deleted file mode 100644 index da6b253db..000000000 --- a/cactus/engine/engine_model.cpp +++ /dev/null @@ -1,1102 +0,0 @@ -#include "engine.h" -#include "../models/model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - - -Model::Model() - : graph_handle_(nullptr), - config_(), - tokenizer_(nullptr), - initialized_(false), - attention_scale_(0.0f), - output_weight_node_id_(0), - owns_graph_(false) { -} - -Model::Model(const Config& config) - : graph_handle_(nullptr), - config_(config), - tokenizer_(nullptr), - initialized_(false), - attention_scale_(0.0f), - output_weight_node_id_(0), - owns_graph_(false) { -} - -Model::~Model() { - if (graph_handle_ && owns_graph_) { - delete static_cast(graph_handle_); - } -} - -bool Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (initialized_) { - return true; - } - auto* gb = new CactusGraph(); - graph_handle_ = gb; - owns_graph_ = true; - embedding_file_path_ = model_folder + "/token_embeddings.weights"; - return init_internal(gb, model_folder, context_size, system_prompt, do_warmup); -} - -bool Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!external_graph) { - throw std::invalid_argument("External graph pointer must not be null"); - } - if (initialized_) { - graph_handle_ = external_graph; - owns_graph_ = false; - return true; - } - - owns_graph_ = false; - graph_handle_ = external_graph; - return init_internal(external_graph, model_folder, context_size, system_prompt, do_warmup); -} - -bool Model::init_internal(CactusGraph* gb, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - - CACTUS_LOG_DEBUG("model", "Initializing model from: " << model_folder); - model_folder_path_ = model_folder; - std::string config_path = model_folder + "/config.txt"; - - if (!config_.from_json(config_path)) { - CACTUS_LOG_ERROR("model", "Model initialization failed - config not loaded from: " << model_folder); - return false; - } - - std::string vocab_file = model_folder + "/vocab.txt"; - std::string merges_file = model_folder + "/merges.txt"; - std::string tokenizer_config_file = model_folder + "/tokenizer_config.txt"; - TokenizerRuntimeConfig tokenizer_runtime_config = load_tokenizer_runtime_config(tokenizer_config_file); - - std::ifstream merges_check(merges_file); - bool has_merges = false; - if (merges_check.is_open()) { - std::string line; - int line_count = 0; - while (std::getline(merges_check, line) && line_count < 10) { - if (!line.empty() && line[0] != '#') { - has_merges = true; - break; - } - line_count++; - } - merges_check.close(); - } - - if (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::BPE || - (tokenizer_runtime_config.tokenizer_type == TokenizerRuntimeConfig::TokenizerType::UNKNOWN && has_merges)) { - tokenizer_ = std::make_unique(); - } else { - tokenizer_ = std::make_unique(); - } - - if (!tokenizer_->load_vocabulary_with_config(vocab_file, merges_file, tokenizer_config_file)) { - return false; - } - - graph_handle_ = gb; - - if(config_.model_type == Config::ModelType::WHISPER){ - embedding_file_path_ = model_folder+"/decoder_token_embeddings.weights"; - } - else{ - embedding_file_path_ = model_folder + "/token_embeddings.weights"; - } - - load_weights_to_graph(gb); - - if (config_.model_type == Config::ModelType::GEMMA3N || config_.model_type == Config::ModelType::GEMMA4) { - attention_scale_ = 1.0f; - } else if (config_.model_type == Config::ModelType::GEMMA) { - attention_scale_ = 1.0f / std::sqrt(256.0f); - } else { - attention_scale_ = 1.0f / std::sqrt(static_cast(config_.attention_head_dim)); - } - - Precision cache_precision = (config_.model_type == Config::ModelType::WHISPER || - config_.model_type == Config::ModelType::MOONSHINE || - config_.model_type == Config::ModelType::PARAKEET || - config_.model_type == Config::ModelType::PARAKEET_TDT || - config_.model_type == Config::ModelType::NEEDLE) - ? Precision::FP16 - : Precision::INT8; - kv_cache_.init(config_.num_layers, context_size, get_kv_layer_dims(), get_kv_layer_heads(), cache_precision); - - size_t window_size = std::min(context_size, size_t(512)); - size_t sink_size = 4; - const char* env_window = std::getenv("CACTUS_KV_WINDOW_SIZE"); - const char* env_sink = std::getenv("CACTUS_KV_SINK_SIZE"); - if (env_window) { - window_size = std::stoul(env_window); - } - if (env_sink) { - sink_size = std::stoul(env_sink); - } - kv_cache_.set_window_size(window_size, sink_size); - cache_k_output_nodes_.resize(config_.num_layers); - cache_v_output_nodes_.resize(config_.num_layers); - - post_init(); - - initialized_ = true; - - if (do_warmup && - config_.model_type != Config::ModelType::WHISPER && - config_.model_type != Config::ModelType::MOONSHINE && - config_.model_type != Config::ModelType::PARAKEET && - config_.model_type != Config::ModelType::PARAKEET_TDT && - config_.model_type != Config::ModelType::NEEDLE) { - std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; - auto warmup_tokens = tokenizer_->encode(warmup_text); - if (config_.model_type == Config::ModelType::GEMMA4) { - warmup_tokens = {2}; - } - forward(warmup_tokens); - auto* gb = static_cast(graph_handle_); - gb->execute(); - } - - reset_cache(); - return true; -} - -size_t Model::forward(const std::vector& /*mel_bins*/, const std::vector& tokens, bool use_cache){ - return forward(tokens, use_cache); -} - -void Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) { - return; - } - - if (has_npu_prefill()) { - size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); - if (tokens.size() > npu_chunk_size) { - prefill_npu(tokens); - return; - } - } - - auto* gb = static_cast(graph_handle_); - - auto process_chunk = [&](const std::vector& chunk) { - forward(chunk, true); - gb->execute(profile_file); - post_execute_updates(gb, chunk.size()); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - - for (size_t chunk_idx = 0; chunk_idx < num_full_chunks; ++chunk_idx) { - size_t start = chunk_idx * chunk_size; - size_t end = start + chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + end); - if (chunk_idx == 1) { - gb->set_prefill_mode(true); - } - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - std::vector final_chunk(tokens.begin() + final_start, tokens.end()); - process_chunk(final_chunk); -} - -void Model::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file) { - (void)image_paths; - prefill(tokens, get_prefill_chunk_size(), profile_file); -} - -uint32_t Model::decode(const std::vector& tokens, float temperature, float top_p, - size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - auto final_hidden = forward(tokens, true); - - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto last_hidden = gb->index(final_hidden, tokens.size() - 1, 0); - const auto& last_hidden_buf = gb->get_output_buffer(last_hidden); - size_t hidden_dim = last_hidden_buf.shape[0]; - last_hidden = gb->reshape(last_hidden, {1, hidden_dim}); - - auto logits_node_id = gb->matmul(last_hidden, output_weight_node_id_, true, backend); - - if (config_.final_logit_softcapping > 0.0f) { - float inv_cap = 1.0f / config_.final_logit_softcapping; - logits_node_id = gb->scalar_multiply(logits_node_id, inv_cap); - logits_node_id = gb->tanh(logits_node_id); - logits_node_id = gb->scalar_multiply(logits_node_id, config_.final_logit_softcapping); - } - auto sampled_token_id = sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty); - - gb->execute(profile_file); - - compute_entropy(gb, logits_node_id, out_entropy); - - post_execute_updates(gb, tokens.size()); - update_kv_cache(gb, tokens.size()); - - auto* output_ptr = gb->get_output(sampled_token_id); - uint32_t result_token = *static_cast(output_ptr); - record_sampled_token(result_token); - return result_token; -} - -size_t Model::sample_token(CactusGraph* gb, size_t logits_node_id, float temperature, float top_p, size_t top_k, - float min_p, float repetition_penalty, - const std::unordered_map* extra_bias) const { - auto combined_bias = tool_constrainer_.get_bias(); - for (const auto& [token_id, boost] : vocab_bias_) { - combined_bias[token_id] += boost; - } - if (extra_bias) { - for (const auto& [token_id, boost] : *extra_bias) { - combined_bias[token_id] += boost; - } - } - if (!token_history_.empty() && repetition_penalty > 1.0f && std::isfinite(repetition_penalty)) { - float log_penalty = std::log(repetition_penalty); - for (uint32_t tok : token_history_) { - combined_bias[tok] -= log_penalty; - } - } - return gb->sample_with_options(logits_node_id, temperature, top_p, min_p, 1.0f, top_k, combined_bias); -} - -void Model::compute_entropy(CactusGraph* gb, size_t logits_node_id, float* out_entropy) { - if (!out_entropy) return; - - const auto& logits_buf = gb->get_output_buffer(logits_node_id); - void* logits_ptr = gb->get_output(logits_node_id); - size_t vocab_size = logits_buf.shape.back(); - size_t seq_len = 1; - if (logits_buf.shape.size() >= 2) - seq_len = logits_buf.shape[logits_buf.shape.size() - 2]; - size_t row_offset = (seq_len > 0 ? (seq_len - 1) * vocab_size : 0); - - std::vector logits(vocab_size); - if (logits_buf.precision == Precision::FP32) { - float* src = static_cast(logits_ptr) + row_offset; - std::copy(src, src + vocab_size, logits.begin()); - } else if (logits_buf.precision == Precision::FP16) { - __fp16* src = static_cast<__fp16*>(logits_ptr) + row_offset; - Quantization::fp16_to_fp32(src, logits.data(), vocab_size); - } else { - int8_t* src = static_cast(logits_ptr) + row_offset; - Quantization::int8_to_fp32(src, logits.data(), vocab_size, 1.0f); - } - - float max_logit = *std::max_element(logits.begin(), logits.end()); - double sum_exp = 0.0; - for (size_t i = 0; i < vocab_size; ++i) - sum_exp += std::exp(static_cast(logits[i] - max_logit)); - double log_sum_exp = static_cast(max_logit) + std::log(sum_exp); - - double entropy = 0.0; - for (size_t i = 0; i < vocab_size; ++i) { - double log_prob = static_cast(logits[i]) - log_sum_exp; - double prob = std::exp(log_prob); - if (prob > 1e-10) - entropy -= prob * log_prob; - } - - double max_entropy = std::log(static_cast(vocab_size)); - *out_entropy = static_cast(entropy / max_entropy); -} - -uint32_t Model::decode_with_audio(const std::vector& tokens, const std::vector& /*mel_bins*/, float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty, - float* /*out_token_time_start*/, float* /*out_token_time_end*/){ - return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -uint32_t Model::decode_with_images(const std::vector& tokens, const std::vector& image_paths, - float temperature, float top_p, size_t top_k, const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - (void)image_paths; - return decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -std::vector Model::get_image_embeddings(const std::string& /*image_path*/) { - throw std::runtime_error("Image embeddings not supported for this model type"); -} - -std::vector Model::get_audio_embeddings(const std::vector& /*mel_bins*/) { - throw std::runtime_error("Audio embeddings not supported for this model type"); -} - -void Model::update_kv_cache(CactusGraph* gb, size_t seq_len) { - kv_cache_.update_from_graph(gb, cache_k_output_nodes_, cache_v_output_nodes_, - seq_len, config_.num_layers); -} - -void Model::remove_thinking_tokens(const std::vector>& ranges) { - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) - kv_cache_.remove_token_range(it->first, it->second); -} - -std::vector Model::get_embeddings(const std::vector& tokens, bool pooled, bool normalize, const std::string& profile_file) { - std::vector embeddings; - auto final_hidden = forward(tokens); - - auto* gb = static_cast(graph_handle_); - auto* output_ptr = gb->get_output(final_hidden); - const auto& output_buffer = gb->get_output_buffer(final_hidden); - - if (pooled) { - auto pooled_hidden = gb->mean(final_hidden, 0); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - post_execute_updates(gb, tokens.size()); - auto* pooled_ptr = gb->get_output(pooled_hidden); - const auto& pooled_buffer = gb->get_output_buffer(pooled_hidden); - - size_t hidden_dim = pooled_buffer.total_size; - embeddings.resize(hidden_dim); - - if (pooled_buffer.precision == Precision::FP32) { - float* pooled_data = static_cast(pooled_ptr); - std::copy(pooled_data, pooled_data + hidden_dim, embeddings.begin()); - } else if (pooled_buffer.precision == Precision::FP16) { - __fp16* pooled_data = static_cast<__fp16*>(pooled_ptr); - Quantization::fp16_to_fp32(pooled_data, embeddings.data(), hidden_dim); - } else if (pooled_buffer.precision == Precision::INT8) { - int8_t* pooled_data = static_cast(pooled_ptr); - Quantization::int8_to_fp32(pooled_data, embeddings.data(), hidden_dim, 1.0f); - } - } else { - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - post_execute_updates(gb, tokens.size()); - - size_t total_size = output_buffer.total_size; - embeddings.resize(total_size); - - if (output_buffer.precision == Precision::FP32) { - float* hidden_states = static_cast(output_ptr); - std::copy(hidden_states, hidden_states + total_size, embeddings.begin()); - } else if (output_buffer.precision == Precision::FP16) { - __fp16* hidden_states = static_cast<__fp16*>(output_ptr); - for (size_t i = 0; i < total_size; i++) { - embeddings[i] = static_cast(hidden_states[i]); - } - } else if (output_buffer.precision == Precision::INT8) { - int8_t* hidden_states = static_cast(output_ptr); - for (size_t i = 0; i < total_size; i++) { - embeddings[i] = static_cast(hidden_states[i]); - } - } - } - - if (normalize && !embeddings.empty()) { - float norm_sq = 0.0f; - for (float v : embeddings) { - norm_sq += v * v; - } - float norm = std::sqrt(norm_sq); - if (norm > 1e-12f) { - float inv_norm = 1.0f / norm; - for (float& v : embeddings) { - v *= inv_norm; - } - } - } - - kv_cache_.reset(); - - return embeddings; -} - -bool Config::from_json(const std::string& config_path) { - std::ifstream file(config_path); - if (!file) { - CACTUS_LOG_ERROR("config", "Failed to open config file: " << config_path); - return false; - } - - std::string line; - while (std::getline(file, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = line.substr(0, eq_pos); - std::string value = line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "vocab_size") vocab_size = static_cast(std::stoul(value)); - else if (key == "bos_token_id") bos_token_id = static_cast(std::stoul(value)); - else if (key == "eos_token_id") eos_token_id = static_cast(std::stoul(value)); - else if (key == "num_layers") num_layers = static_cast(std::stoul(value)); - else if (key == "hidden_dim") hidden_dim = static_cast(std::stoul(value)); - else if (key == "ffn_intermediate_dim") ffn_intermediate_dim = static_cast(std::stoul(value)); - else if (key == "attention_heads") attention_heads = static_cast(std::stoul(value)); - else if (key == "attention_kv_heads") attention_kv_heads = static_cast(std::stoul(value)); - else if (key == "attention_head_dim") attention_head_dim = static_cast(std::stoul(value)); - else if (key == "layer_norm_eps") layer_norm_eps = std::stof(value); - else if (key == "rope_theta") rope_theta = std::stof(value); - else if (key == "num_experts") num_experts = static_cast(std::stoul(value)); - else if (key == "num_shared_experts") num_shared_experts = static_cast(std::stoul(value)); - else if (key == "num_top_experts") num_top_experts = static_cast(std::stoul(value)); - else if (key == "moe_every_n_layers") moe_every_n_layers = static_cast(std::stoul(value)); - else if (key == "moe_intermediate_dim" || key == "moe_intermediate_size") moe_intermediate_dim = static_cast(std::stoul(value)); - else if (key == "num_dense_layers") num_dense_layers = static_cast(std::stoul(value)); - else if (key == "num_experts_per_tok") num_experts_per_tok = static_cast(std::stoul(value)); - else if (key == "norm_topk_prob") norm_topk_prob = (value == "true" || value == "1"); - else if (key == "use_expert_bias") use_expert_bias = (value == "true" || value == "1"); - else if (key == "routed_scaling_factor") routed_scaling_factor = std::stof(value); - else if (key == "tie_word_embeddings") tie_word_embeddings = (value == "true" || value == "1"); - else if (key == "vision_hidden_dim" || key == "vision_hidden_size") vision_hidden_dim = static_cast(std::stoul(value)); - else if (key == "vision_num_layers") vision_num_layers = static_cast(std::stoul(value)); - else if (key == "vision_attention_heads") vision_attention_heads = static_cast(std::stoul(value)); - else if (key == "vision_image_size") vision_image_size = static_cast(std::stoul(value)); - else if (key == "vision_patch_size") vision_patch_size = static_cast(std::stoul(value)); - else if (key == "vision_num_channels") vision_num_channels = static_cast(std::stoul(value)); - else if (key == "vision_embed_dim") vision_embed_dim = static_cast(std::stoul(value)); - else if (key == "visual_tokens_per_img") visual_tokens_per_img = static_cast(std::stoul(value)); - else if (key == "use_pixel_shuffle") use_pixel_shuffle = (value == "true" || value == "1"); - else if (key == "pixel_shuffle_factor") pixel_shuffle_factor = static_cast(std::stoul(value)); - else if (key == "use_image_tokens") use_image_tokens = (value == "true" || value == "1"); - else if (key == "image_token_id") image_token_id = static_cast(std::stoul(value)); - else if (key == "use_layout_tags") use_layout_tags = (value == "true" || value == "1"); - else if (key == "image_seq_len") image_seq_len = static_cast(std::stoul(value)); - else if (key == "global_image_size") global_image_size = static_cast(std::stoul(value)); - else if (key == "max_tile_size") max_tile_size = static_cast(std::stoul(value)); - else if (key == "rescale_factor") rescale_factor = std::stof(value); - else if (key == "image_mean") image_mean = std::stof(value); - else if (key == "image_std") image_std = std::stof(value); - else if (key == "downsample_factor") downsample_factor = static_cast(std::stoul(value)); - else if (key == "min_tiles") min_tiles = static_cast(std::stoul(value)); - else if (key == "max_tiles") max_tiles = static_cast(std::stoul(value)); - else if (key == "use_thumbnail") use_thumbnail = (value == "true" || value == "1"); - else if (key == "min_image_tokens") min_image_tokens = static_cast(std::stoul(value)); - else if (key == "max_image_tokens") max_image_tokens = static_cast(std::stoul(value)); - else if (key == "tile_size") tile_size = static_cast(std::stoul(value)); - else if (key == "max_pixels_tolerance") max_pixels_tolerance = std::stof(value); - else if (key == "do_image_splitting") do_image_splitting = (value == "true" || value == "1"); - else if (key == "precision") { - if (value == "INT8") precision = Precision::INT8; - else if (value == "FP16") precision = Precision::FP16; - else precision = Precision::FP32; - } - else if (key == "model_type") { - std::string model_type_value = value; - std::transform(model_type_value.begin(), model_type_value.end(), model_type_value.begin(), ::tolower); - if (value == "gemma" || value == "GEMMA") model_type = ModelType::GEMMA; - else if (value == "lfm2" || value == "LFM2" || value == "lfm2_moe" || value == "LFM2_MOE") model_type = ModelType::LFM2; - else if (value == "bert" || value == "BERT") model_type = ModelType::NOMIC; - else if (value == "whisper" || value == "WHISPER") model_type = ModelType::WHISPER; - else if (value == "moonshine" || value == "MOONSHINE") model_type = ModelType::MOONSHINE; - else if (value == "silero_vad" || value == "SILERO_VAD") model_type = ModelType::SILERO_VAD; - else if (value == "parakeet" || value == "PARAKEET") model_type = ModelType::PARAKEET; - else if (model_type_value.rfind("qwen3_5", 0) == 0) model_type = ModelType::QWEN3P5; - else if (value == "parakeet_tdt" || value == "PARAKEET_TDT") model_type = ModelType::PARAKEET_TDT; - else if (value == "gemma3n" || value == "GEMMA3N") model_type = ModelType::GEMMA3N; - else if (value == "needle" || value == "NEEDLE") model_type = ModelType::NEEDLE; - else if (value == "gemma4" || value == "GEMMA4" || value == "tinyllama" || value == "TINYLLAMA") model_type = ModelType::GEMMA4; - else if (value == "youtu" || value == "YOUTU") model_type = ModelType::YOUTU; - else if (value == "pyannote" || value == "PYANNOTE") model_type = ModelType::PYANNOTE; - else if (value == "wespeaker" || value == "WESPEAKER") model_type = ModelType::WESPEAKER; - else model_type = ModelType::QWEN; - } - else if (key == "model_variant") { - std::string v = value; - std::transform(v.begin(), v.end(), v.begin(), ::tolower); - if (v == "vlm") model_variant = ModelVariant::VLM; - else if (v == "extract") model_variant = ModelVariant::EXTRACT; - else if (v == "rag") model_variant = ModelVariant::RAG; - else model_variant = ModelVariant::DEFAULT; - } - else if (key == "conv_L_cache") conv_L_cache = static_cast(std::stoul(value)); - else if (key == "layer_types") { - layer_types.clear(); - std::string sanitized; - sanitized.reserve(value.size()); - for (char c : value) { - if (c == '[' || c == ']' || c == '\'' || c == '"') { - continue; - } - sanitized.push_back(c); - } - std::stringstream ss(sanitized); - std::string item; - while (std::getline(ss, item, ',')) { - if (!item.empty()) { - item.erase(0, item.find_first_not_of(" \t")); - item.erase(item.find_last_not_of(" \t") + 1); - if (!item.empty()) layer_types.push_back(item); - } - } - } - else if (key == "enc_hidden_act") encoder_act_gelu = (value == "gelu"); - else if (key == "dec_hidden_act") decoder_act_gelu = (value == "gelu"); - else if (key == "num_encoder_layers") num_encoder_layers = static_cast(std::stoul(value)); - else if (key == "num_decoder_layers") num_decoder_layers = static_cast(std::stoul(value)); - else if (key == "partial_rotary_factor") partial_rotary_factor = std::stof(value); - else if (key == "pad_token_id") pad_token_id = static_cast(std::stoul(value)); - else if (key == "conv_kernel_size") conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_kernel_size") subsampling_conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_stride") subsampling_conv_stride = static_cast(std::stoul(value)); - else if (key == "subsampling_conv_channels") subsampling_conv_channels = static_cast(std::stoul(value)); - else if (key == "subsampling_factor") subsampling_factor = static_cast(std::stoul(value)); - else if (key == "num_mel_bins") num_mel_bins = static_cast(std::stoul(value)); - else if (key == "encoder_hidden_act") encoder_hidden_act = value; - else if (key == "linear_num_key_heads") linear_num_key_heads = static_cast(std::stoul(value)); - else if (key == "linear_key_head_dim") linear_key_head_dim = static_cast(std::stoul(value)); - else if (key == "linear_num_value_heads") linear_num_value_heads = static_cast(std::stoul(value)); - else if (key == "linear_value_head_dim") linear_value_head_dim = static_cast(std::stoul(value)); - else if (key == "linear_q_proj_dim") linear_q_proj_dim = static_cast(std::stoul(value)); - else if (key == "kv_lora_rank") kv_lora_rank = static_cast(std::stoul(value)); - else if (key == "q_lora_rank") q_lora_rank = static_cast(std::stoul(value)); - else if (key == "qk_head_dim") qk_head_dim = static_cast(std::stoul(value)); - else if (key == "qk_nope_head_dim") qk_nope_head_dim = static_cast(std::stoul(value)); - else if (key == "qk_rope_head_dim") qk_rope_head_dim = static_cast(std::stoul(value)); - else if (key == "v_head_dim") v_head_dim = static_cast(std::stoul(value)); - else if (key == "rope_interleave") rope_interleave = (value == "true" || value == "1"); - else if (key == "attention_bias") attention_bias = (value == "true" || value == "1"); - else if (key == "rope_scaling_factor") rope_scaling_factor = std::stof(value); - else if (key == "rope_mscale_all_dim") rope_mscale_all_dim = std::stof(value); - else if (key == "linear_k_proj_dim") linear_k_proj_dim = static_cast(std::stoul(value)); - else if (key == "linear_v_proj_dim") linear_v_proj_dim = static_cast(std::stoul(value)); - else if (key == "predictor_hidden_dim") predictor_hidden_dim = static_cast(std::stoul(value)); - else if (key == "predictor_num_layers") predictor_num_layers = static_cast(std::stoul(value)); - else if (key == "tdt_joint_dim") tdt_joint_dim = static_cast(std::stoul(value)); - else if (key == "tdt_num_durations") tdt_num_durations = static_cast(std::stoul(value)); - else if (key == "tdt_blank_id") tdt_blank_id = static_cast(std::stoul(value)); - else if (key == "tdt_durations") { - tdt_durations.clear(); - std::stringstream ss(value); - std::string item; - while (std::getline(ss, item, ',')) { - size_t first = item.find_first_not_of(" \t"); - if (first == std::string::npos) continue; - size_t last = item.find_last_not_of(" \t"); - item = item.substr(first, last - first + 1); - tdt_durations.push_back(static_cast(std::stoul(item))); - } - } - else if (key == "altup_num_inputs") altup_num_inputs = static_cast(std::stoul(value)); - else if (key == "laurel_rank") laurel_rank = static_cast(std::stoul(value)); - else if (key == "hidden_size_per_layer_input") hidden_size_per_layer_input = static_cast(std::stoul(value)); - else if (key == "num_kv_shared_layers") num_kv_shared_layers = static_cast(std::stoul(value)); - else if (key == "sliding_window") sliding_window = static_cast(std::stoul(value)); - else if (key == "rope_local_base_freq") rope_local_base_freq = std::stof(value); - else if (key == "final_logit_softcapping") final_logit_softcapping = std::stof(value); - else if (key == "global_partial_rotary_factor") global_partial_rotary_factor = std::stof(value); - else if (key == "expert_intermediate_size") expert_intermediate_size = static_cast(std::stoul(value)); - else if (key == "global_head_dim") global_head_dim = static_cast(std::stoul(value)); - else if (key == "num_global_kv_heads" || key == "num_global_key_value_heads") num_global_kv_heads = static_cast(std::stoul(value)); - else if (key == "attention_k_eq_v") attention_k_eq_v = (value == "true" || value == "1"); - else if (key == "enable_moe_block") enable_moe_block = (value == "true" || value == "1"); - else if (key == "vision_head_dim") vision_head_dim = static_cast(std::stoul(value)); - else if (key == "vision_kv_heads") vision_kv_heads = static_cast(std::stoul(value)); - else if (key == "vision_intermediate_size") vision_intermediate_size = static_cast(std::stoul(value)); - else if (key == "vision_position_embedding_size") vision_position_embedding_size = static_cast(std::stoul(value)); - else if (key == "vision_pooling_kernel_size") vision_pooling_kernel_size = static_cast(std::stoul(value)); - else if (key == "vision_default_output_length") vision_default_output_length = static_cast(std::stoul(value)); - else if (key == "vision_rope_theta") vision_rope_theta = std::stof(value); - else if (key == "audio_hidden_dim") audio_hidden_dim = static_cast(std::stoul(value)); - else if (key == "audio_num_layers") audio_num_layers = static_cast(std::stoul(value)); - else if (key == "audio_num_heads") audio_num_heads = static_cast(std::stoul(value)); - else if (key == "audio_head_dim") audio_head_dim = static_cast(std::stoul(value)); - else if (key == "audio_input_feat_size") audio_input_feat_size = static_cast(std::stoul(value)); - else if (key == "audio_conf_conv_kernel_size") audio_conf_conv_kernel_size = static_cast(std::stoul(value)); - else if (key == "audio_chunk_size") audio_chunk_size = static_cast(std::stoul(value)); - else if (key == "audio_context_left") audio_context_left = static_cast(std::stoul(value)); - else if (key == "audio_context_right") audio_context_right = static_cast(std::stoul(value)); - else if (key == "audio_logit_cap") audio_logit_cap = std::stof(value); - else if (key == "audio_residual_weight") audio_residual_weight = std::stof(value); - else if (key == "audio_output_proj_dims") audio_output_proj_dims = static_cast(std::stoul(value)); - else if (key == "audio_vocab_size") audio_vocab_size = static_cast(std::stoul(value)); - else if (key == "audio_vocab_offset") audio_vocab_offset = static_cast(std::stoul(value)); - else if (key == "audio_soft_tokens") audio_soft_tokens = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv0_channels") audio_sscp_conv0_channels = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv1_channels") audio_sscp_conv1_channels = static_cast(std::stoul(value)); - else if (key == "audio_sscp_conv_eps") audio_sscp_conv_eps = std::stof(value); - else if (key == "audio_rms_norm_eps") audio_rms_norm_eps = std::stof(value); - else if (key == "audio_fft_length") audio_fft_length = static_cast(std::stoul(value)); - else if (key == "audio_fft_overdrive") { - audio_fft_overdrive = (value == "true" || value == "1"); - audio_fft_length = audio_fft_overdrive ? 1024u : 512u; - } - else if (key == "audio_token_id") audio_token_id = static_cast(std::stoul(value)); - else if (key == "channel_open_token_id") channel_open_token_id = static_cast(std::stoul(value)); - else if (key == "channel_close_token_id") channel_close_token_id = static_cast(std::stoul(value)); - else if (key == "activation_sparsity_ppf") { - activation_sparsity_ppf.clear(); - std::stringstream ss(value); - std::string item; - while (std::getline(ss, item, ',')) { - size_t first = item.find_first_not_of(" \t"); - if (first == std::string::npos) continue; - size_t last = item.find_last_not_of(" \t"); - item = item.substr(first, last - first + 1); - activation_sparsity_ppf.push_back(std::stof(item)); - } - } - } - - if (is_gemma_family(model_type)) { - default_temperature = 1.0f; - default_top_p = 0.95f; - default_top_k = 64; - if (model_type == ModelType::GEMMA4) { - default_cloud_handoff_threshold = 0.92f; - default_rolling_entropy_window = 16; - } - } else if (model_type == ModelType::LFM2) { - default_temperature = 0.3f; - default_top_p = 0.95f; - default_top_k = 20; - } else if (model_type == ModelType::QWEN) { - default_temperature = 0.6f; - default_top_p = 0.95f; - default_top_k = 20; - } else if (model_type == ModelType::QWEN3P5) { - default_temperature = 0.7f; - default_top_p = 0.8f; - default_top_k = 20; - } else if (model_type == ModelType::WHISPER) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_cloud_handoff_threshold = 0.4f; - } else if (model_type == ModelType::MOONSHINE) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 6.5f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::PARAKEET) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 8.0f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::PARAKEET_TDT) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - default_max_tps = 8.0f; - default_cloud_handoff_threshold = 0.35f; - } else if (model_type == ModelType::NEEDLE) { - default_temperature = 0.0f; - default_top_p = 0.0f; - default_top_k = 0; - } else if (model_type == ModelType::YOUTU) { - default_temperature = 1.0f; - default_top_p = 0.95f; - default_top_k = 20; - } - - return true; -} - -std::string Config::to_json() const { - return "{}"; -} - -std::unique_ptr create_model(const std::string& model_folder) { - CACTUS_LOG_DEBUG("model", "Creating model from: " << model_folder); - Config config; - std::string config_path = model_folder + "/config.txt"; - - if (!config.from_json(config_path)) { - CACTUS_LOG_ERROR("model", "Failed to create model - cannot load config from: " << model_folder); - return nullptr; - } - - const bool has_vision_support = - config.use_image_tokens || - config.vision_num_layers > 0 || - config.vision_embed_dim > 0 || - config.vision_hidden_dim > 0 || - config.visual_tokens_per_img > 0; - - if (config.model_type == Config::ModelType::LFM2 && has_vision_support) { - return std::make_unique(config); - } - - const bool has_audio_support = - config.audio_num_layers > 0 || - config.audio_hidden_dim > 0; - - if (config.model_type == Config::ModelType::GEMMA4 && (has_vision_support || has_audio_support)) { - return std::make_unique(config); - } - - switch (config.model_type) { - case Config::ModelType::QWEN: - return std::make_unique(config); - case Config::ModelType::QWEN3P5: - return std::make_unique(config); - case Config::ModelType::GEMMA: - return std::make_unique(config); - case Config::ModelType::GEMMA3N: - return std::make_unique(config); - case Config::ModelType::LFM2: - if (config.num_experts > 0 && config.moe_intermediate_dim > 0 && config.num_experts_per_tok > 0) { - return std::make_unique(config); - } - return std::make_unique(config); - case Config::ModelType::NOMIC: - return std::make_unique(config); - case Config::ModelType::WHISPER: - return std::make_unique(config); - case Config::ModelType::MOONSHINE: - return std::make_unique(config); - case Config::ModelType::SILERO_VAD: - return std::make_unique(config); - case Config::ModelType::PARAKEET: - return std::make_unique(config); - case Config::ModelType::PARAKEET_TDT: - return std::make_unique(config); - case Config::ModelType::NEEDLE: - return std::make_unique(config); - case Config::ModelType::GEMMA4: - return std::make_unique(config); - case Config::ModelType::YOUTU: - return std::make_unique(config); - case Config::ModelType::PYANNOTE: - return std::make_unique(config); - case Config::ModelType::WESPEAKER: - return std::make_unique(config); - default: - return std::make_unique(config); - } -} - -void Model::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) const { - auto* graph = static_cast(graph_handle_); - if (!graph) { - return; - } - graph->capture_debug_node(layer_idx, name, node_id); -} - -void Model::clear_debug_nodes() { - auto* graph = static_cast(graph_handle_); - if (!graph) { - return; - } - graph->clear_debug_nodes(); -} - -const std::vector& Model::get_debug_nodes() const { - auto* graph = static_cast(graph_handle_); - debug_nodes_.clear(); - if (!graph) { - return debug_nodes_; - } - - const auto& entries = graph->get_debug_nodes(); - debug_nodes_.reserve(entries.size()); - for (const auto& entry : entries) { - debug_nodes_.push_back({entry.layer_idx, entry.name, entry.node_id}); - } - return debug_nodes_; -} - -bool Model::load_npu_prefill(const std::string& model_path) { - CACTUS_LOG_DEBUG("npu", "Attempting to load NPU prefill from: " << model_path); - - npu_prefill_ = npu::create_prefill(); - if (!npu_prefill_) { - CACTUS_LOG_DEBUG("npu", "NPU prefill creation failed (not supported on this device)"); - return false; - } - - bool loaded = npu_prefill_->load(model_path); - if (loaded) { - CACTUS_LOG_INFO("npu", "NPU prefill loaded successfully from: " << model_path); - } else { - CACTUS_LOG_DEBUG("npu", "NPU prefill model not found at: " << model_path); - } - return loaded; -} - -bool Model::has_npu_prefill() const { - return npu_prefill_ && npu_prefill_->is_available(); -} - -size_t Model::get_prefill_chunk_size() const { - if (has_npu_prefill()) { - return static_cast(npu_prefill_->get_chunk_size()); - } - return 256; // default chunk size -} - -std::vector<__fp16> Model::get_token_embeddings(const std::vector& tokens) { - auto* gb = static_cast(graph_handle_); - if (!gb || tokens.empty()) { - return {}; - } - - gb->soft_reset(); - - size_t tok_input = gb->input({tokens.size()}, Precision::FP32); - std::vector tok_f(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) { - tok_f[i] = static_cast(tokens[i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - - size_t embedding_node = gb->embedding(embedding_node_id_, tok_input); - - gb->execute(); - - const auto& emb_buf = gb->get_output_buffer(embedding_node); - void* emb_ptr = gb->get_output(embedding_node); - - size_t num_tokens = tokens.size(); - size_t hidden_dim = config_.hidden_dim; - std::vector<__fp16> embeddings(num_tokens * hidden_dim); - - if (emb_buf.precision == Precision::FP16) { - __fp16* src = static_cast<__fp16*>(emb_ptr); - std::copy(src, src + num_tokens * hidden_dim, embeddings.begin()); - } else if (emb_buf.precision == Precision::FP32) { - float* src = static_cast(emb_ptr); - for (size_t i = 0; i < num_tokens * hidden_dim; i++) { - embeddings[i] = static_cast<__fp16>(src[i]); - } - } else if (emb_buf.precision == Precision::INT8) { - int8_t* src = static_cast(emb_ptr); - for (size_t i = 0; i < num_tokens * hidden_dim; i++) { - embeddings[i] = static_cast<__fp16>(src[i]); - } - } - - return embeddings; -} - -void Model::prefill_npu(const std::vector& tokens) { - if (!npu_prefill_ || !npu_prefill_->is_available()) { - throw std::runtime_error("NPU prefill not available"); - } - - const int chunk_size = npu_prefill_->get_chunk_size(); - const int hidden_dim = npu_prefill_->get_hidden_dim(); - const int num_layers = npu_prefill_->get_num_layers(); - const int fallback_num_kv_heads = npu_prefill_->get_num_kv_heads(); - const int fallback_head_dim = npu_prefill_->get_head_dim(); - - const std::vector layer_dims = get_kv_layer_dims(); - const std::vector layer_heads = get_kv_layer_heads(); - const int layers_to_update = std::min(num_layers, static_cast(config_.num_layers)); - - std::vector<__fp16> all_embeddings = get_token_embeddings(tokens); - if (all_embeddings.empty()) { - throw std::runtime_error("Failed to get token embeddings for NPU prefill"); - } - - if (Config::is_gemma_family(config_.model_type)) { - float scale = std::sqrt(static_cast(hidden_dim)); - for (size_t i = 0; i < all_embeddings.size(); i++) { - all_embeddings[i] = __fp16(static_cast(all_embeddings[i]) * scale); - } - } - - size_t num_tokens = tokens.size(); - size_t num_chunks = (num_tokens + chunk_size - 1) / chunk_size; - - for (size_t c = 0; c < num_chunks; c++) { - size_t start = c * chunk_size; - size_t actual_tokens = std::min(static_cast(chunk_size), num_tokens - start); - - std::vector<__fp16> chunk_embeddings(chunk_size * hidden_dim, __fp16(0)); - std::copy(all_embeddings.begin() + start * hidden_dim, - all_embeddings.begin() + (start + actual_tokens) * hidden_dim, - chunk_embeddings.begin()); - - int position_offset = static_cast(start); - - npu::NPUPrefillDirectResult direct_result = npu_prefill_->prefill_chunk_direct(chunk_embeddings, position_offset); - - if (direct_result.valid) { - for (int layer_idx = 0; layer_idx < layers_to_update; layer_idx++) { - const auto& k_ref = direct_result.k_caches[layer_idx]; - const auto& v_ref = direct_result.v_caches[layer_idx]; - - if (k_ref.data && v_ref.data) { - size_t layer_kv_heads = layer_idx < static_cast(layer_heads.size()) - ? layer_heads[layer_idx] - : static_cast(fallback_num_kv_heads); - size_t layer_head_dim = layer_idx < static_cast(layer_dims.size()) - ? layer_dims[layer_idx] - : static_cast(fallback_head_dim); - - size_t expected = static_cast(chunk_size) * layer_kv_heads * layer_head_dim; - if (expected > 0 && (k_ref.count < expected || v_ref.count < expected)) { - CACTUS_LOG_WARN( - "npu", - "NPU prefill cache output too small for layer " << layer_idx - << " (expected>=" << expected - << ", got k=" << k_ref.count << ", v=" << v_ref.count << "); skipping layer"); - continue; - } - - kv_cache_.update_from_npu(layer_idx, k_ref.data, v_ref.data, - actual_tokens, layer_kv_heads, layer_head_dim); - } - } - } - } -} - -double Model::score_tokens_window_logprob( - const std::vector& tokens, - size_t start, - size_t end, - size_t context, - size_t* tokens_scored -) { - if (tokens_scored) - *tokens_scored = 0; - - if (tokens.empty()) - return 0.0; - - if (end > tokens.size()) - end = tokens.size(); - - if (start >= end) - return 0.0; - - if (start == 0) - start = 1; - - if (start >= end) - return 0.0; - - const size_t target_len = end - start; - const size_t ctx_begin = (start > context) ? (start - context) : 0; - - if (end < 2) return 0.0; - const size_t input_end = end - 1; - - if (input_end <= ctx_begin) - return 0.0; - - std::vector input_tokens(tokens.begin() + ctx_begin,tokens.begin() + input_end); - - if (tokens_scored) - *tokens_scored = target_len; - - reset_cache(); - - auto* gb = static_cast(graph_handle_); - const auto backend = (config_.default_backend == Config::Backend::CPU) ? ComputeBackend::CPU : ComputeBackend::NPU; - - const size_t hidden_node = forward(input_tokens, /*use_cache=*/false); - const auto& hidden_buf = gb->get_output_buffer(hidden_node); - - if (hidden_buf.shape.size() != 2) { - throw std::runtime_error("Expected hidden to be rank-2 [L, hidden_dim]"); - } - - - const size_t first_pos = start - ctx_begin - 1; - const size_t hidden_slice = gb->slice(hidden_node, /*axis=*/0, first_pos, target_len); - bool transpose_w = true; - const size_t logits_node = gb->matmul(hidden_slice, output_weight_node_id_, transpose_w, backend); - gb->execute(); - - const auto& logits_buf = gb->get_output_buffer(logits_node); - if (logits_buf.shape.size() != 2) - throw std::runtime_error("Expected logits to be rank-2 [T, vocab]"); - - const size_t T = logits_buf.shape[0]; - const size_t vocab_size = logits_buf.shape[1]; - - if (T != target_len) - throw std::runtime_error("Logits T dimension does not match target_len"); - - void* logits_ptr = gb->get_output(logits_node); - std::vector row(vocab_size); - double total_logprob = 0.0; - - for (size_t i = 0; i < target_len; ++i) { - const uint32_t y = tokens[start + i]; - if (y >= vocab_size) - throw std::runtime_error("Target token out of vocab range"); - - if (logits_buf.precision == Precision::FP32) { - const float* src = static_cast(logits_ptr) + i * vocab_size; - std::memcpy(row.data(), src, vocab_size * sizeof(float)); - } - else if (logits_buf.precision == Precision::FP16) { - const __fp16* src = static_cast(logits_ptr) + i * vocab_size; - Quantization::fp16_to_fp32(const_cast<__fp16*>(src), row.data(), vocab_size); - } - else { - const int8_t* src = static_cast(logits_ptr) + i * vocab_size; - Quantization::int8_to_fp32(const_cast(src), row.data(), vocab_size, 1.0f); - } - - float max_logit = *std::max_element(row.begin(), row.end()); - double sum = 0.0; - - for (size_t j = 0; j < vocab_size; ++j) - sum += std::exp(double(row[j] - max_logit)); - - const double lse = double(max_logit) + std::log(sum); - total_logprob += double(row[y]) - lse; - } - - return total_logprob; -} -} -} diff --git a/cactus/engine/engine_signals.cpp b/cactus/engine/engine_signals.cpp deleted file mode 100644 index 38468b4c0..000000000 --- a/cactus/engine/engine_signals.cpp +++ /dev/null @@ -1,775 +0,0 @@ -#include "engine.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -#ifndef M_PI -#define M_PI 3.14159265358979323846 -#endif - -namespace cactus { -namespace engine { - -static void to_db( - float* spectrogram, - size_t size, - float reference, - float min_value, - const float* db_range, - float multiplier) -{ - if (reference <= 0.0f) { - throw std::invalid_argument("reference must be greater than zero"); - } - if (min_value <= 0.0f) { - throw std::invalid_argument("min_value must be greater than zero"); - } - - reference = std::max(min_value, reference); - const float log_ref = std::log10(reference); - - CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - float value = std::max(min_value, spectrogram[i]); - spectrogram[i] = multiplier * (std::log10(value) - log_ref); - } - }); - - if (db_range != nullptr) { - if (*db_range <= 0.0f) { - throw std::invalid_argument("db_range must be greater than zero"); - } - - float max_db = CactusThreading::parallel_reduce, float, std::function>( - size, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) { - float local_max = -std::numeric_limits::infinity(); - for (size_t i = start; i < end; i++) { - local_max = std::max(local_max, spectrogram[i]); - } - return local_max; - }, - -std::numeric_limits::infinity(), - [](float a, float b) { return std::max(a, b); } - ); - - float min_db = max_db - *db_range; - CactusThreading::parallel_for(size, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::max(min_db, spectrogram[i]); - } - }); - } -} - -static size_t bit_reverse(size_t x, size_t log2n) { - size_t result = 0; - for (size_t i = 0; i < log2n; i++) { - result = (result << 1) | (x & 1); - x >>= 1; - } - return result; -} - -static bool is_power_of_two(size_t n) { - return n != 0 && (n & (n - 1)) == 0; -} - -static size_t log2_power_of_two(size_t n) { - size_t log2n = 0; - for (size_t temp = n; temp > 1; temp >>= 1) { - log2n++; - } - return log2n; -} - -static void fft_radix2(float* re, float* im, size_t n, bool inverse) { - if (!is_power_of_two(n)) return; - - const size_t log2n = log2_power_of_two(n); - - for (size_t i = 0; i < n; i++) { - size_t j = bit_reverse(i, log2n); - if (i < j) { - std::swap(re[i], re[j]); - std::swap(im[i], im[j]); - } - } - - for (size_t s = 1; s <= log2n; s++) { - size_t m = size_t{1} << s; - size_t m2 = m >> 1; - float w_re = 1.0f; - float w_im = 0.0f; - const float angle = static_cast(M_PI) / static_cast(m2); - const float wm_re = std::cos(angle); - const float wm_im = inverse ? std::sin(angle) : -std::sin(angle); - - for (size_t j = 0; j < m2; j++) { - for (size_t k = j; k < n; k += m) { - size_t k_m2 = k + m2; - float t_re = w_re * re[k_m2] - w_im * im[k_m2]; - float t_im = w_re * im[k_m2] + w_im * re[k_m2]; - float u_re = re[k]; - float u_im = im[k]; - re[k] = u_re + t_re; - im[k] = u_im + t_im; - re[k_m2] = u_re - t_re; - im[k_m2] = u_im - t_im; - } - float new_w_re = w_re * wm_re - w_im * wm_im; - float new_w_im = w_re * wm_im + w_im * wm_re; - w_re = new_w_re; - w_im = new_w_im; - } - } -} - -enum class FFTNorm { - BACKWARD, - FORWARD, - ORTHO -}; - -static FFTNorm parse_fft_norm(const char* norm) { - if (!norm || std::strcmp(norm, "backward") == 0) { - return FFTNorm::BACKWARD; - } - if (std::strcmp(norm, "forward") == 0) { - return FFTNorm::FORWARD; - } - if (std::strcmp(norm, "ortho") == 0) { - return FFTNorm::ORTHO; - } - throw std::invalid_argument("norm must be one of {\"backward\",\"forward\",\"ortho\"}"); -} - -static size_t next_power_of_two(size_t n) { - size_t padded_n = 1; - while (padded_n < n) { - padded_n <<= 1; - } - return padded_n; -} - -static float fft_norm_factor(size_t n, FFTNorm norm_mode, bool inverse) { - if (norm_mode == FFTNorm::ORTHO) { - return 1.0f / std::sqrt(static_cast(n)); - } - if (norm_mode == FFTNorm::FORWARD) { - return inverse ? 1.0f : (1.0f / static_cast(n)); - } - return inverse ? (1.0f / static_cast(n)) : 1.0f; -} - -static void ensure_irfft_scratch(std::vector& re, std::vector& im, size_t n) { - if (re.size() < n) { - re.resize(n); - } - if (im.size() < n) { - im.resize(n); - } -} - -static void idft_r_f32_1d_dft( - const float* input, - float* output, - const size_t n, - const float norm_factor) { - const size_t in_len = n / 2 + 1; - const float two_pi_over_n = (2.0f * static_cast(M_PI)) / static_cast(n); - for (size_t t = 0; t < n; ++t) { - float sum = input[0]; - for (size_t k = 1; k < in_len; ++k) { - const float re = input[k * 2]; - const float im = input[k * 2 + 1]; - const float angle = two_pi_over_n * static_cast(k * t); - const float c = std::cos(angle); - const float s = std::sin(angle); - const bool self_conjugate = (k * 2 == n); - if (self_conjugate) { - sum += re * c; - } else { - sum += 2.0f * (re * c - im * s); - } - } - output[t] = sum * norm_factor; - } -} - -static void rfft_f32_1d(const float* input, float* output, const size_t n, const char* norm) { - const size_t out_len = n / 2 + 1; - const FFTNorm norm_mode = parse_fft_norm(norm); - const float norm_factor = fft_norm_factor(n, norm_mode, false); - - if (n == 1) { - output[0] = input[0] * norm_factor; - output[1] = 0.0f; - return; - } - -#ifdef __APPLE__ - { - size_t log2n = 0; - size_t padded_n = 1; - while (padded_n < n) { - padded_n <<= 1; - log2n++; - } - - FFTSetup fft_setup = vDSP_create_fftsetup(log2n, FFT_RADIX2); - if (fft_setup) { - std::vector buffer(padded_n, 0.0f); - std::copy(input, input + n, buffer.begin()); - - DSPSplitComplex split; - std::vector real_part(padded_n / 2); - std::vector imag_part(padded_n / 2); - split.realp = real_part.data(); - split.imagp = imag_part.data(); - - vDSP_ctoz(reinterpret_cast(buffer.data()), 2, &split, 1, padded_n / 2); - - vDSP_fft_zrip(fft_setup, &split, 1, log2n, FFT_FORWARD); - - float scale = 0.5f * norm_factor; - - output[0] = split.realp[0] * scale * 2.0f; - output[1] = 0.0f; - - if (out_len > 1) { - size_t nyquist_idx = padded_n / 2; - if (nyquist_idx < out_len) { - output[nyquist_idx * 2] = split.imagp[0] * scale * 2.0f; - output[nyquist_idx * 2 + 1] = 0.0f; - } - } - - for (size_t i = 1; i < out_len && i < padded_n / 2; i++) { - output[i * 2] = split.realp[i] * scale * 2.0f; - output[i * 2 + 1] = split.imagp[i] * scale * 2.0f; - } - - vDSP_destroy_fftsetup(fft_setup); - return; - } - } -#endif - - const size_t padded_n = next_power_of_two(n); - - std::vector re(padded_n, 0.0f), im(padded_n, 0.0f); - std::copy(input, input + n, re.begin()); - - fft_radix2(re.data(), im.data(), padded_n, false); - - for (size_t i = 0; i < out_len; i++) { - output[i * 2] = re[i] * norm_factor; - output[i * 2 + 1] = im[i] * norm_factor; - } -} - -static void idft_r_f32_1d(const float* input, float* output, const size_t n, const char* norm) { - const size_t in_len = n / 2 + 1; - const FFTNorm norm_mode = parse_fft_norm(norm); - const float norm_factor = fft_norm_factor(n, norm_mode, true); - - if (n == 1) { - output[0] = input[0] * norm_factor; - return; - } - - if (!is_power_of_two(n)) { - idft_r_f32_1d_dft(input, output, n, norm_factor); - return; - } - - const size_t padded_n = next_power_of_two(n); - const size_t padded_half_plus_one = padded_n / 2 + 1; - thread_local std::vector re_scratch; - thread_local std::vector im_scratch; - ensure_irfft_scratch(re_scratch, im_scratch, padded_n); - std::fill(re_scratch.begin(), re_scratch.begin() + padded_n, 0.0f); - std::fill(im_scratch.begin(), im_scratch.begin() + padded_n, 0.0f); - float* re = re_scratch.data(); - float* im = im_scratch.data(); - for (size_t i = 0; i < in_len; ++i) { - re[i] = input[i * 2]; - im[i] = input[i * 2 + 1]; - } - - im[0] = 0.0f; - const bool has_padded_nyquist_bin = in_len == padded_half_plus_one; - if (has_padded_nyquist_bin) { - im[padded_n / 2] = 0.0f; - } - - const size_t mirror_limit = has_padded_nyquist_bin ? (in_len - 1) : in_len; - for (size_t i = 1; i < mirror_limit; ++i) { - const size_t mirrored = padded_n - i; - re[mirrored] = re[i]; - im[mirrored] = -im[i]; - } - - fft_radix2(re, im, padded_n, true); - - for (size_t i = 0; i < n; ++i) { - output[i] = re[i] * norm_factor; - } -} - -static float hertz_to_mel(float freq, const char* mel_scale) { - if (std::strcmp(mel_scale, "htk") == 0) { - return 2595.0f * std::log10(1.0f + (freq / 700.0f)); - } else if (std::strcmp(mel_scale, "kaldi") == 0) { - return 1127.0f * std::log(1.0f + (freq / 700.0f)); - } - - const float min_log_hertz = 1000.0f; - const float min_log_mel = 15.0f; - const float logstep = 27.0f / std::log(6.4f); - float mels = 3.0f * freq / 200.0f; - - if (freq >= min_log_hertz) { - mels = min_log_mel + std::log(freq / min_log_hertz) * logstep; - } - - return mels; -} - -static float mel_to_hertz(float mels, const char* mel_scale) { - if (std::strcmp(mel_scale, "htk") == 0) { - return 700.0f * (std::pow(10.0f, mels / 2595.0f) - 1.0f); - } else if (std::strcmp(mel_scale, "kaldi") == 0) { - return 700.0f * (std::exp(mels / 1127.0f) - 1.0f); - } - - const float min_log_hertz = 1000.0f; - const float min_log_mel = 15.0f; - const float logstep = std::log(6.4f) / 27.0f; - float freq = 200.0f * mels / 3.0f; - - if (mels >= min_log_mel) { - freq = min_log_hertz * std::exp(logstep * (mels - min_log_mel)); - } - - return freq; -} - -static void generate_mel_filter_bank( - float* mel_filters, - const int num_frequency_bins, - const int num_mel_filters, - const float min_frequency, - const float max_frequency, - const int sampling_rate, - const char* norm, - const char* mel_scale, - const bool triangularize_in_mel_space) -{ - if (norm != nullptr && std::strcmp(norm, "slaney") != 0) { - throw std::invalid_argument("norm must be one of None or \"slaney\""); - } - - if (std::strcmp(mel_scale, "htk") != 0 && std::strcmp(mel_scale, "kaldi") != 0 && std::strcmp(mel_scale, "slaney") != 0) { - throw std::invalid_argument("mel_scale should be one of \"htk\", \"slaney\" or \"kaldi\"."); - } - - if (num_frequency_bins < 2) { - throw std::invalid_argument( - "Require num_frequency_bins: " + std::to_string(num_frequency_bins) + " >= 2"); - } - - if (min_frequency > max_frequency) { - throw std::invalid_argument( - "Require min_frequency: " + std::to_string(min_frequency) + - " <= max_frequency: " + std::to_string(max_frequency)); - } - - const float mel_min = hertz_to_mel(min_frequency, mel_scale); - const float mel_max = hertz_to_mel(max_frequency, mel_scale); - - std::vector mel_freqs(num_mel_filters + 2); - for (int i = 0; i < num_mel_filters + 2; i++) { - mel_freqs[i] = mel_min + (mel_max - mel_min) * i / (num_mel_filters + 1); - } - - std::vector filter_freqs(num_mel_filters + 2); - for (int i = 0; i < num_mel_filters + 2; i++) { - filter_freqs[i] = mel_to_hertz(mel_freqs[i], mel_scale); - } - - std::vector fft_freqs(num_frequency_bins); - if (triangularize_in_mel_space) { - float fft_bin_width = static_cast(sampling_rate) / ((num_frequency_bins - 1) * 2); - for (int i = 0; i < num_frequency_bins; i++) { - fft_freqs[i] = hertz_to_mel(fft_bin_width * i, mel_scale); - } - filter_freqs = mel_freqs; - } else { - for (int i = 0; i < num_frequency_bins; i++) { - fft_freqs[i] = (static_cast(sampling_rate) / 2.0f) * i / (num_frequency_bins - 1); - } - } - - for (int i = 0; i < num_mel_filters; i++) { - float left_edge = filter_freqs[i]; - float center = filter_freqs[i + 1]; - float right_edge = filter_freqs[i + 2]; - - for (int j = 0; j < num_frequency_bins; j++) { - float freq = fft_freqs[j]; - float down_slope = (freq - left_edge) / (center - left_edge); - float up_slope = (right_edge - freq) / (right_edge - center); - - mel_filters[i * num_frequency_bins + j] = std::max(0.0f, std::min(down_slope, up_slope)); - } - } - - if (norm != nullptr && std::strcmp(norm, "slaney") == 0) { - for (int i = 0; i < num_mel_filters; i++) { - float enorm = 2.0f / (filter_freqs[i + 2] - filter_freqs[i]); - for (int j = 0; j < num_frequency_bins; j++) { - mel_filters[i * num_frequency_bins + j] *= enorm; - } - } - } -} - -static void compute_spectrogram_f32( - const float* waveform, - size_t waveform_length, - const float* window, - size_t window_length, - size_t frame_length, - size_t hop_length, - const size_t* fft_length, - float* spectrogram, - float power, - bool center, - const char* pad_mode, - bool onesided [[maybe_unused]], - float dither, - const float* preemphasis, - const float* mel_filters, - size_t mel_filters_size, - float mel_floor, - const char* log_mel, - float reference, - float min_value, - const float* db_range, - bool remove_dc_offset) -{ - size_t actual_fft_length; - if (fft_length == nullptr) { - actual_fft_length = frame_length; - } else { - actual_fft_length = *fft_length; - } - - if (frame_length > actual_fft_length) { - throw std::invalid_argument( - "frame_length (" + std::to_string(frame_length) + - ") may not be larger than fft_length (" + - std::to_string(actual_fft_length) + ")"); - } - - std::vector hann_window; - const float* actual_window = window; - - if (window == nullptr) { - size_t length = frame_length + 1; - hann_window.resize(frame_length); - for (size_t i = 0; i < frame_length; i++) { - hann_window[i] = 0.5f * (1.0f - std::cos(2.0f * static_cast(M_PI) * i / (length - 1))); - } - actual_window = hann_window.data(); - } else if (window_length != frame_length) { - throw std::invalid_argument( - "Length of the window (" + std::to_string(window_length) + - ") must equal frame_length (" + std::to_string(frame_length) + ")"); - } - - if (hop_length <= 0) { - throw std::invalid_argument("hop_length must be greater than zero"); - } - - if (power == 0.0f && mel_filters != nullptr) { - throw std::invalid_argument( - "You have provided `mel_filters` but `power` is `None`. " - "Mel spectrogram computation is not yet supported for complex-valued spectrogram. " - "Specify `power` to fix this issue."); - } - - std::vector padded_waveform; - const float* input_waveform = waveform; - size_t input_length = waveform_length; - - if (center) { - size_t pad_length = frame_length / 2; - size_t padded_length = waveform_length + 2 * pad_length; - padded_waveform.assign(padded_length, 0.0f); - - if (std::strcmp(pad_mode, "reflect") == 0) { - for (size_t i = 0; i < pad_length; i++) { - padded_waveform[i] = waveform[pad_length - i]; - } - - std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); - - for (size_t i = 0; i < pad_length; i++) { - padded_waveform[pad_length + waveform_length + i] = waveform[waveform_length - 2 - i]; - } - } else if (std::strcmp(pad_mode, "constant") == 0) { - std::copy(waveform, waveform + waveform_length, padded_waveform.data() + pad_length); - } else { - throw std::invalid_argument("Unsupported pad_mode: " + std::string(pad_mode)); - } - - input_waveform = padded_waveform.data(); - input_length = padded_length; - } - - const size_t num_frames = 1 + (input_length - frame_length) / hop_length; - const size_t num_frequency_bins = (actual_fft_length / 2) + 1; - - std::vector buffer(actual_fft_length); - std::vector raw_complex_frequencies(num_frequency_bins * 2); - - const size_t num_mel_bins = mel_filters != nullptr ? mel_filters_size / num_frequency_bins : 0; - const size_t spectrogram_bins = mel_filters != nullptr ? num_mel_bins : num_frequency_bins; - - std::vector temp_spectrogram(num_frames * num_frequency_bins); - - CactusThreading::parallel_for(num_frames, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [&](size_t start_frame, size_t end_frame) { - std::vector local_buffer(actual_fft_length); - std::vector local_complex_frequencies(num_frequency_bins * 2); - - for (size_t frame_idx = start_frame; frame_idx < end_frame; frame_idx++) { - size_t timestep = frame_idx * hop_length; - std::fill(local_buffer.begin(), local_buffer.end(), 0.0f); - - size_t available_length = std::min(frame_length, input_length - timestep); - std::copy(input_waveform + timestep, input_waveform + timestep + available_length, local_buffer.data()); - - if (dither != 0.0f) { - thread_local std::mt19937 rng(std::random_device{}()); - std::normal_distribution dist(0.0f, 1.0f); - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] += dither * dist(rng); - } - } - - if (remove_dc_offset) { - float mean = 0.0f; - for (size_t i = 0; i < frame_length; i++) { - mean += local_buffer[i]; - } - mean /= static_cast(frame_length); - - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] -= mean; - } - } - - if (preemphasis != nullptr) { - float preemph_coef = *preemphasis; - for (size_t i = frame_length - 1; i > 0; i--) { - local_buffer[i] -= preemph_coef * local_buffer[i - 1]; - } - local_buffer[0] *= (1.0f - preemph_coef); - } - - for (size_t i = 0; i < frame_length; i++) { - local_buffer[i] *= actual_window[i]; - } - - rfft_f32_1d(local_buffer.data(), local_complex_frequencies.data(), actual_fft_length, "backward"); - - for (size_t i = 0; i < num_frequency_bins; i++) { - float real = local_complex_frequencies[i * 2]; - float imag = local_complex_frequencies[i * 2 + 1]; - float magnitude = std::hypot(real, imag); - temp_spectrogram[frame_idx * num_frequency_bins + i] = std::pow(magnitude, power); - } - } - }); - - if (mel_filters != nullptr) { - CactusThreading::parallel_for_2d(num_mel_bins, num_frames, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t m, size_t t) { - float sum = 0.0f; - for (size_t f = 0; f < num_frequency_bins; f++) { - sum += mel_filters[m * num_frequency_bins + f] * temp_spectrogram[t * num_frequency_bins + f]; - } - spectrogram[m * num_frames + t] = std::max(mel_floor, sum); - }); - } else { - CactusThreading::parallel_for_2d(num_frames, num_frequency_bins, CactusThreading::Thresholds::AXIS_REDUCE, [&](size_t t, size_t f) { - spectrogram[f * num_frames + t] = temp_spectrogram[t * num_frequency_bins + f]; - }); - } - - if (power != 0.0f && log_mel != nullptr) { - const size_t total_elements = spectrogram_bins * num_frames; - - if (std::strcmp(log_mel, "log") == 0) { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::log(spectrogram[i]); - } - }); - } else if (std::strcmp(log_mel, "log10") == 0) { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ALL_REDUCE, [&](size_t start, size_t end) { - for (size_t i = start; i < end; i++) { - spectrogram[i] = std::log10(spectrogram[i]); - } - }); - } else if (std::strcmp(log_mel, "dB") == 0) { - if (power == 1.0f) { - to_db(spectrogram, total_elements, reference, min_value, db_range, 20.0f); - } else if (power == 2.0f) { - to_db(spectrogram, total_elements, reference, min_value, db_range, 10.0f); - } else { - throw std::invalid_argument( - "Cannot use log_mel option 'dB' with power " + std::to_string(power)); - } - } else { - throw std::invalid_argument("Unknown log_mel option: " + std::string(log_mel)); - } - } -} - -AudioProcessor::AudioProcessor() - : num_frequency_bins_(0), num_mel_filters_(0) {} - -AudioProcessor::~AudioProcessor() = default; - -void AudioProcessor::init_mel_filters(size_t num_frequency_bins, - size_t num_mel_filters, - float min_freq, - float max_freq, - size_t sampling_rate, - const char* norm, - const char* mel_scale) { - num_frequency_bins_ = num_frequency_bins; - num_mel_filters_ = num_mel_filters; - mel_filters_.resize(num_mel_filters * num_frequency_bins); - - generate_mel_filter_bank( - mel_filters_.data(), - num_frequency_bins, - num_mel_filters, - min_freq, - max_freq, - sampling_rate, - norm, - mel_scale, - false - ); -} - -std::vector AudioProcessor::compute_spectrogram( - const std::vector& waveform, - const SpectrogramConfig& config) { - - if (mel_filters_.empty()) { - throw std::runtime_error("Mel filters not initialized. Call init_mel_filters() first."); - } - - const size_t n_samples = waveform.size(); - const size_t fft_size = config.fft_override > 0 ? config.fft_override : config.n_fft; - const size_t analysis_frame_length = config.n_fft; - const size_t window_length = std::min(config.frame_length, analysis_frame_length); - const size_t pad_length = config.center ? analysis_frame_length / 2 : 0; - const size_t padded_length = n_samples + 2 * pad_length; - if (analysis_frame_length == 0 || padded_length < analysis_frame_length || config.hop_length == 0) { - return {}; - } - const size_t num_frames = 1 + (padded_length - analysis_frame_length) / config.hop_length; - - std::vector output(num_mel_filters_ * num_frames); - std::vector window(analysis_frame_length, 0.0f); - if (window_length == 1) { - window[analysis_frame_length / 2] = 1.0f; - } else if (window_length > 1) { - const float denom = config.hann_periodic - ? static_cast(window_length) - : static_cast(window_length - 1); - const float a0 = config.window_a0; - const size_t left_pad = (analysis_frame_length - window_length) / 2; - for (size_t i = 0; i < window_length; ++i) { - const float w = a0 - (1.0f - a0) * std::cos(2.0f * static_cast(M_PI) * static_cast(i) / denom); - window[left_pad + i] = w; - } - } - - float effective_floor = config.mel_floor_additive ? 0.0f : config.mel_floor; - const char* effective_log = config.mel_floor_additive ? nullptr : config.log_mel; - - compute_spectrogram_f32( - waveform.data(), - waveform.size(), - window.data(), - window.size(), - analysis_frame_length, - config.hop_length, - &fft_size, - output.data(), - config.power, - config.center, - config.pad_mode, - config.onesided, - config.dither, - config.preemphasis != 0.0f ? &config.preemphasis : nullptr, - mel_filters_.data(), - mel_filters_.size(), - effective_floor, - effective_log, - config.reference, - config.min_value, - nullptr, - config.remove_dc_offset - ); - - if (config.mel_floor_additive) { - for (size_t i = 0; i < output.size(); i++) { -#ifdef __APPLE__ - // vDSP FFT returns 2x standard DFT magnitudes, so raw mel - // values are 2x too large. Halve before adding floor and log. - output[i] = std::log(output[i] * 0.5f + config.mel_floor); -#else - output[i] = std::log(output[i] + config.mel_floor); -#endif - } - } - - return output; -} - -std::vector AudioProcessor::compute_irfft( - const std::vector& complex_input, - size_t n, - const char* norm) { - if (n == 0) { - throw std::invalid_argument("compute_irfft: n must be greater than 0"); - } - const size_t expected_in_len = (n / 2 + 1) * 2; - if (complex_input.size() != expected_in_len) { - throw std::invalid_argument("compute_irfft: complex_input must contain exactly (n/2+1)*2 values"); - } - std::vector output(n); - idft_r_f32_1d(complex_input.data(), output.data(), n, norm); - return output; -} - -} -} diff --git a/cactus/engine/engine_sp.cpp b/cactus/engine/engine_sp.cpp deleted file mode 100644 index a80754747..000000000 --- a/cactus/engine/engine_sp.cpp +++ /dev/null @@ -1,507 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -SPTokenizer::SPTokenizer() - : trie_root_(std::make_unique()), - vocab_size_(0), - unk_token_id_(3), - bos_token_id_(2), - eos_token_id_(1), - pad_token_id_(0), - vocab_mmap_ptr_(nullptr), - vocab_mmap_size_(0) { - has_chat_template_ = false; -} - -SPTokenizer::~SPTokenizer() { - cleanup_mmap(); -} - -void SPTokenizer::cleanup_mmap() { - if (vocab_mmap_ptr_ && vocab_mmap_ptr_ != MAP_FAILED) { - munmap(vocab_mmap_ptr_, vocab_mmap_size_); - vocab_mmap_ptr_ = nullptr; - } -} - -bool SPTokenizer::load_vocabulary_with_config(const std::string& vocab_file, const std::string& /*merges_file*/, const std::string& config_file) { - runtime_config_ = load_tokenizer_runtime_config(config_file); - std::string config_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/config.txt"; - detect_model_type(config_path); - - std::ifstream vocab_stream(vocab_file); - if (!vocab_stream.is_open()) return false; - - token_to_id_.clear(); - id_to_token_.clear(); - token_scores_.clear(); - - std::string first_line; - std::getline(vocab_stream, first_line); - vocab_stream.seekg(0); - - bool is_id_token_format = false; - if (!first_line.empty()) { - is_id_token_format = (std::isdigit(first_line[0]) && - first_line.find('\t') != std::string::npos); - } - - if (is_id_token_format) { - std::string line = ""; - while (std::getline(vocab_stream, line)) { - std::string token = ""; - uint32_t id = UINT32_MAX; - - std::istringstream iss(line); - if (iss >> id) { - if (std::getline(iss, token)) { - if (!token.empty() && token[0] == '\t') { - token = token.substr(1); - } - } - - if (token.empty()) { - auto last_pos = vocab_stream.tellg(); - while (std::getline(vocab_stream, line)) { - if (!line.empty()) { - break; - } - token += '\n'; - last_pos = vocab_stream.tellg(); - } - vocab_stream.seekg(last_pos); - } - } - - if (!token.empty() && id != UINT32_MAX) { - float score = -static_cast(id); - size_t tab_in_token = token.find('\t'); - if (tab_in_token != std::string::npos) { - std::string score_str = token.substr(tab_in_token + 1); - if (!score_str.empty()) score = std::stof(score_str); - token = token.substr(0, tab_in_token); - } - token_to_id_[token] = id; - if (id >= id_to_token_.size()) { - id_to_token_.resize(id + 1); - token_scores_.resize(id + 1, 0.0f); - } - id_to_token_[id] = token; - token_scores_[id] = score; - } - } - vocab_size_ = id_to_token_.size(); - } else { - std::string line; - uint32_t id = 0; - - vocab_stream.seekg(0); - while (std::getline(vocab_stream, line)) { - token_to_id_[line] = id; - id_to_token_.push_back(line); - token_scores_.push_back(0.0f); - id++; - } - vocab_size_ = id; - } - - vocab_stream.close(); - - build_trie(); - - std::ifstream config_stream(config_file); - if (config_stream.is_open()) { - std::string config_line; - while (std::getline(config_stream, config_line)) { - if (config_line.empty() || config_line[0] == '#') continue; - - size_t eq_pos = config_line.find('='); - if (eq_pos == std::string::npos) continue; - - std::string key = config_line.substr(0, eq_pos); - std::string value = config_line.substr(eq_pos + 1); - - key.erase(0, key.find_first_not_of(" \t")); - key.erase(key.find_last_not_of(" \t") + 1); - value.erase(0, value.find_first_not_of(" \t")); - value.erase(value.find_last_not_of(" \t") + 1); - - if (key == "eos_token_id") { - eos_token_id_ = std::stoul(value); - } else if (key == "pad_token_id") { - pad_token_id_ = std::stoul(value); - } else if (key == "unk_token_id") { - unk_token_id_ = std::stoul(value); - } else if (key == "bos_token_id") { - bos_token_id_ = std::stoul(value); - } else if (key == "sp_model_type") { - sp_bpe_mode_ = (value == "bpe" || value == "BPE"); - } else if (key == "sp_add_dummy_prefix") { - sp_add_dummy_prefix_ = (value == "true" || value == "1"); - } else if (key == "sp_byte_fallback") { - sp_byte_fallback_ = (value == "true" || value == "1"); - } - } - } - - std::string special_tokens_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/special_tokens.json"; - load_special_tokens(special_tokens_path); - - std::string template_path = config_file.substr(0, config_file.find_last_of("/\\")) + "/chat_template.jinja2"; - load_chat_template(template_path); - - return true; -} - -void SPTokenizer::build_trie() { - for (uint32_t id = 0; id < id_to_token_.size(); ++id) { - const std::string& token = id_to_token_[id]; - if (token.empty()) continue; - - std::u32string u32_token; - size_t pos = 0; - while (pos < token.length()) { - char32_t codepoint = 0; - unsigned char byte = token[pos]; - - if (byte < 0x80) { - codepoint = byte; - pos++; - } else if ((byte & 0xE0) == 0xC0) { - if (pos + 1 < token.length()) { - codepoint = ((byte & 0x1F) << 6) | (token[pos + 1] & 0x3F); - pos += 2; - } else break; - } else if ((byte & 0xF0) == 0xE0) { - if (pos + 2 < token.length()) { - codepoint = ((byte & 0x0F) << 12) | - ((token[pos + 1] & 0x3F) << 6) | - (token[pos + 2] & 0x3F); - pos += 3; - } else break; - } else if ((byte & 0xF8) == 0xF0) { - if (pos + 3 < token.length()) { - codepoint = ((byte & 0x07) << 18) | - ((token[pos + 1] & 0x3F) << 12) | - ((token[pos + 2] & 0x3F) << 6) | - (token[pos + 3] & 0x3F); - pos += 4; - } else break; - } else { - pos++; - continue; - } - - u32_token.push_back(codepoint); - } - - if (u32_token.empty()) continue; - - TrieNode* current = trie_root_.get(); - for (char32_t ch : u32_token) { - if (current->children.find(ch) == current->children.end()) { - current->children[ch] = std::make_unique(); - } - current = current->children[ch].get(); - } - current->token_id = static_cast(id); - current->score = token_scores_[id]; - } -} - -std::string SPTokenizer::preprocess_text(const std::string& text) const { - if (text.empty()) return text; - - if (sp_bpe_mode_) { - std::string processed; - if (sp_add_dummy_prefix_) processed += "\xE2\x96\x81"; - for (char c : text) { - if (c == ' ') processed += "\xE2\x96\x81"; - else processed += c; - } - return processed; - } - - std::string processed = ""; - if (model_type_ == ModelType::BERT) { - processed = "▁"; - } - - for (size_t i = text.find_first_not_of(" "); i < text.length(); i++) { - char c = text[i]; - if (c == ' ') { - processed += "▁"; - } else { - processed += c; - } - } - - return processed; -} - -std::string SPTokenizer::postprocess_text(const std::string& text) const { - std::string result; - size_t i = 0; - while (i < text.length()) { - if (i + 2 < text.length() && - static_cast(text[i]) == 0xE2 && - static_cast(text[i+1]) == 0x96 && - static_cast(text[i+2]) == 0x81) { - result += ' '; - i += 3; - } else { - result += text[i]; - i++; - } - } - if (!result.empty() && result[0] == ' ') { - result = result.substr(1); - } - return result; -} - -std::vector> SPTokenizer::tokenize_with_trie(const std::string& text) const { - std::vector> result; - - std::u32string u32_text; - size_t pos = 0; - while (pos < text.length()) { - char32_t codepoint = 0; - unsigned char byte = text[pos]; - - if (byte < 0x80) { - codepoint = byte; - pos++; - } else if ((byte & 0xE0) == 0xC0) { - if (pos + 1 < text.length()) { - codepoint = ((byte & 0x1F) << 6) | (text[pos + 1] & 0x3F); - pos += 2; - } else break; - } else if ((byte & 0xF0) == 0xE0) { - if (pos + 2 < text.length()) { - codepoint = ((byte & 0x0F) << 12) | - ((text[pos + 1] & 0x3F) << 6) | - (text[pos + 2] & 0x3F); - pos += 3; - } else break; - } else if ((byte & 0xF8) == 0xF0) { - if (pos + 3 < text.length()) { - codepoint = ((byte & 0x07) << 18) | - ((text[pos + 1] & 0x3F) << 12) | - ((text[pos + 2] & 0x3F) << 6) | - (text[pos + 3] & 0x3F); - pos += 4; - } else break; - } else { - pos++; - continue; - } - - u32_text.push_back(codepoint); - } - - if (u32_text.empty()) { - result.push_back({text, unk_token_id_}); - return result; - } - - pos = 0; - while (pos < u32_text.length()) { - TrieNode* current = trie_root_.get(); - size_t best_match_len = 0; - int32_t best_token_id = -1; - - for (size_t len = 0; pos + len < u32_text.length(); ++len) { - char32_t ch = u32_text[pos + len]; - if (current->children.find(ch) == current->children.end()) { - break; - } - current = current->children[ch].get(); - if (current->token_id >= 0) { - best_match_len = len + 1; - best_token_id = current->token_id; - } - } - - if (best_match_len > 0) { - std::u32string u32_token = u32_text.substr(pos, best_match_len); - - std::string token; - for (char32_t cp : u32_token) { - if (cp < 0x80) { - token.push_back(static_cast(cp)); - } else if (cp < 0x800) { - token.push_back(static_cast(0xC0 | (cp >> 6))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } else if (cp < 0x10000) { - token.push_back(static_cast(0xE0 | (cp >> 12))); - token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } else { - token.push_back(static_cast(0xF0 | (cp >> 18))); - token.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); - token.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - token.push_back(static_cast(0x80 | (cp & 0x3F))); - } - } - result.push_back({token, static_cast(best_token_id)}); - pos += best_match_len; - } else { - char32_t cp = u32_text[pos]; - std::string char_str; - if (cp < 0x80) { - char_str.push_back(static_cast(cp)); - } else if (cp < 0x800) { - char_str.push_back(static_cast(0xC0 | (cp >> 6))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } else if (cp < 0x10000) { - char_str.push_back(static_cast(0xE0 | (cp >> 12))); - char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } else { - char_str.push_back(static_cast(0xF0 | (cp >> 18))); - char_str.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); - char_str.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); - char_str.push_back(static_cast(0x80 | (cp & 0x3F))); - } - result.push_back({char_str, unk_token_id_}); - pos++; - } - } - - return result; -} - -std::vector SPTokenizer::tokenize_with_bpe(const std::string& text) const { - std::vector symbols; - for (size_t i = 0; i < text.size(); ) { - size_t char_len = 1; - unsigned char byte = static_cast(text[i]); - if ((byte & 0xE0) == 0xC0) char_len = 2; - else if ((byte & 0xF0) == 0xE0) char_len = 3; - else if ((byte & 0xF8) == 0xF0) char_len = 4; - if (i + char_len <= text.size()) symbols.push_back(text.substr(i, char_len)); - i += char_len; - } - - while (symbols.size() > 1) { - int best_index = -1; - float best_score = -std::numeric_limits::infinity(); - for (size_t i = 0; i + 1 < symbols.size(); ++i) { - auto it = token_to_id_.find(symbols[i] + symbols[i + 1]); - if (it == token_to_id_.end()) continue; - float score = it->second < token_scores_.size() - ? token_scores_[it->second] - : -static_cast(it->second); - if (score > best_score) { best_score = score; best_index = static_cast(i); } - } - if (best_index < 0) break; - symbols[best_index] += symbols[best_index + 1]; - symbols.erase(symbols.begin() + best_index + 1); - } - - std::vector result; - for (const auto& symbol : symbols) { - auto it = token_to_id_.find(symbol); - if (it != token_to_id_.end()) { result.push_back(it->second); continue; } - if (sp_byte_fallback_) { - for (unsigned char b : symbol) { - char buf[7]; std::snprintf(buf, sizeof(buf), "<0x%02X>", b); - auto byte_it = token_to_id_.find(buf); - result.push_back(byte_it != token_to_id_.end() ? byte_it->second : unk_token_id_); - } - continue; - } - result.push_back(unk_token_id_); - } - return result; -} - -std::vector SPTokenizer::split_with_special_tokens(const std::string& text) const { - return cactus::engine::split_with_special_tokens(text, special_tokens_); -} - -std::vector SPTokenizer::encode(const std::string& text) const { - if (text.empty()) return {}; - - auto text_segments = split_with_special_tokens(text); - std::vector token_ids; - - for (const auto& segment : text_segments) { - auto special_it = special_tokens_.find(segment); - if (special_it != special_tokens_.end()) { - token_ids.push_back(special_it->second); - } else { - std::string processed = preprocess_text(segment); - if (processed.empty()) continue; - - if (sp_bpe_mode_) { - auto ids = tokenize_with_bpe(processed); - token_ids.insert(token_ids.end(), ids.begin(), ids.end()); - } else { - auto token_pairs = tokenize_with_trie(processed); - for (const auto& [token, id] : token_pairs) { - token_ids.push_back(id); - } - } - } - } - - return token_ids; -} - -std::string SPTokenizer::decode(const std::vector& tokens) const { - if (tokens.size() == 1) { - if (tokens[0] >= id_to_token_.size()) return {}; - const std::string& piece = id_to_token_[tokens[0]]; - unsigned int byte_val; - if (piece.size() == 6 && std::sscanf(piece.c_str(), "<0x%02X>", &byte_val) == 1) { - return std::string(1, static_cast(byte_val)); - } - std::string result; - for (size_t i = 0; i < piece.length(); ) { - if (i + 2 < piece.length() && - static_cast(piece[i]) == 0xE2 && - static_cast(piece[i+1]) == 0x96 && - static_cast(piece[i+2]) == 0x81) { - result += ' '; - i += 3; - } else { - result += piece[i++]; - } - } - return result; - } - - std::string raw; - for (uint32_t token_id : tokens) { - if (token_id >= id_to_token_.size()) continue; - const std::string& piece = id_to_token_[token_id]; - unsigned int byte_val; - if (piece.size() == 6 && std::sscanf(piece.c_str(), "<0x%02X>", &byte_val) == 1) { - raw.push_back(static_cast(byte_val)); - } else { - raw += piece; - } - } - return postprocess_text(raw); -} - -void SPTokenizer::load_special_tokens(const std::string& config_file) { - load_special_tokens_map(config_file, special_tokens_); -} - -} // namespace engine -} // namespace cactus diff --git a/cactus/engine/engine_tokenizer.cpp b/cactus/engine/engine_tokenizer.cpp deleted file mode 100644 index d0749714f..000000000 --- a/cactus/engine/engine_tokenizer.cpp +++ /dev/null @@ -1,1018 +0,0 @@ -#include "engine.h" -#include -#include -#include -#include -#include - -extern "C" { - #include "../../libs/stb/stb_image.h" -} - -namespace cactus { -namespace engine { - -namespace { - -std::string trim_copy(const std::string& value) { - size_t start = value.find_first_not_of(" \t\r\n"); - if (start == std::string::npos) return ""; - size_t end = value.find_last_not_of(" \t\r\n"); - return value.substr(start, end - start + 1); -} - -TokenizerRuntimeConfig::TokenizerType parse_tokenizer_type(const std::string& value) { - if (value == "bpe") return TokenizerRuntimeConfig::TokenizerType::BPE; - if (value == "sentencepiece") return TokenizerRuntimeConfig::TokenizerType::SENTENCEPIECE; - return TokenizerRuntimeConfig::TokenizerType::UNKNOWN; -} - -TokenizerRuntimeConfig::VocabFormat parse_vocab_format(const std::string& value) { - if (value == "id_tab_token") return TokenizerRuntimeConfig::VocabFormat::ID_TAB_TOKEN; - if (value == "line_token") return TokenizerRuntimeConfig::VocabFormat::LINE_TOKEN; - return TokenizerRuntimeConfig::VocabFormat::UNKNOWN; -} - -TokenizerRuntimeConfig::Normalizer parse_normalizer(const std::string& value) { - if (value == "metaspace") return TokenizerRuntimeConfig::Normalizer::METASPACE; - if (value == "byte_level") return TokenizerRuntimeConfig::Normalizer::BYTE_LEVEL; - return TokenizerRuntimeConfig::Normalizer::NONE; -} - -TokenizerRuntimeConfig::Decoder parse_decoder(const std::string& value) { - if (value == "replace_metaspace") return TokenizerRuntimeConfig::Decoder::REPLACE_METASPACE; - if (value == "byte_level") return TokenizerRuntimeConfig::Decoder::BYTE_LEVEL; - return TokenizerRuntimeConfig::Decoder::NONE; -} - -void skip_json_whitespace(const std::string& json, size_t& pos) { - while (pos < json.size() && std::isspace(static_cast(json[pos]))) { - ++pos; - } -} - -bool extract_added_token_object(const std::string& json, size_t& pos, std::string& out_object) { - skip_json_whitespace(json, pos); - if (pos >= json.size() || json[pos] != '{') { - return false; - } - - size_t start = pos; - size_t depth = 0; - bool in_string = false; - bool escaped = false; - - while (pos < json.size()) { - char c = json[pos++]; - if (escaped) { - escaped = false; - continue; - } - if (c == '\\') { - escaped = true; - continue; - } - if (c == '"') { - in_string = !in_string; - continue; - } - if (in_string) { - continue; - } - if (c == '{') { - ++depth; - } else if (c == '}') { - if (depth == 0) { - return false; - } - --depth; - if (depth == 0) { - out_object = json.substr(start, pos - start); - return true; - } - } - } - - return false; -} - -bool parse_added_token_entry(const std::string& object, std::string& token_content, uint32_t& token_id, - bool& is_special) { - token_content.clear(); - token_id = 0; - is_special = false; - - size_t id_key = object.find("\"id\""); - if (id_key == std::string::npos) { - return false; - } - size_t id_colon = object.find(':', id_key); - if (id_colon == std::string::npos) { - return false; - } - size_t id_pos = id_colon + 1; - skip_json_whitespace(object, id_pos); - size_t id_end = id_pos; - while (id_end < object.size() && std::isdigit(static_cast(object[id_end]))) { - ++id_end; - } - if (id_end == id_pos) { - return false; - } - token_id = static_cast(std::stoul(object.substr(id_pos, id_end - id_pos))); - - size_t content_key = object.find("\"content\""); - if (content_key == std::string::npos) { - return false; - } - size_t content_colon = object.find(':', content_key); - if (content_colon == std::string::npos) { - return false; - } - size_t content_pos = object.find('"', content_colon + 1); - if (content_pos == std::string::npos) { - return false; - } - ++content_pos; - token_content = extract_json_string(object, content_pos); - - size_t special_key = object.find("\"special\""); - if (special_key != std::string::npos) { - size_t special_colon = object.find(':', special_key); - if (special_colon != std::string::npos) { - size_t special_pos = special_colon + 1; - skip_json_whitespace(object, special_pos); - is_special = object.compare(special_pos, 4, "true") == 0; - } - } - - return true; -} - -void load_tokenizer_json_added_special_tokens( - const std::string& tokenizer_json_path, - std::unordered_map& special_tokens) { - std::ifstream file(tokenizer_json_path); - if (!file.is_open()) { - return; - } - - std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - size_t pos = content.find("\"added_tokens\""); - if (pos == std::string::npos) { - return; - } - - pos = content.find('[', pos); - if (pos == std::string::npos) { - return; - } - ++pos; - - while (pos < content.size()) { - skip_json_whitespace(content, pos); - if (pos >= content.size() || content[pos] == ']') { - break; - } - - std::string object; - if (!extract_added_token_object(content, pos, object)) { - ++pos; - continue; - } - - std::string token_content; - uint32_t token_id = 0; - bool is_special = false; - if (parse_added_token_entry(object, token_content, token_id, is_special) && is_special) { - special_tokens[token_content] = token_id; - } - - skip_json_whitespace(content, pos); - if (pos < content.size() && content[pos] == ',') { - ++pos; - } - } -} - -} // namespace - -TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file) { - TokenizerRuntimeConfig config; - - std::ifstream file(config_file); - if (!file.is_open()) { - return config; - } - - std::string line; - while (std::getline(file, line)) { - if (line.empty() || line[0] == '#') continue; - - size_t eq_pos = line.find('='); - if (eq_pos == std::string::npos) continue; - - const std::string key = trim_copy(line.substr(0, eq_pos)); - const std::string value = trim_copy(line.substr(eq_pos + 1)); - - if (key == "tokenizer_type") { - config.tokenizer_type = parse_tokenizer_type(value); - } else if (key == "vocab_format") { - config.vocab_format = parse_vocab_format(value); - } else if (key == "normalizer") { - config.normalizer = parse_normalizer(value); - } else if (key == "decoder") { - config.decoder = parse_decoder(value); - } else if (key == "byte_fallback") { - config.byte_fallback = (value == "true" || value == "1"); - } else if (key == "has_chat_template") { - config.has_chat_template = (value == "true" || value == "1"); - } - } - - return config; -} - -void load_special_tokens_map(const std::string& config_file, std::unordered_map& special_tokens) { - special_tokens.clear(); - - std::ifstream file(config_file); - if (file.is_open()) { - std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - - size_t pos = content.find("\"special_tokens\""); - if (pos != std::string::npos) { - pos = content.find("{", pos); - if (pos != std::string::npos) { - size_t end_pos = content.find("}", pos); - if (end_pos != std::string::npos) { - std::string special_tokens_section = content.substr(pos + 1, end_pos - pos - 1); - std::istringstream iss(special_tokens_section); - std::string line; - - while (std::getline(iss, line)) { - size_t colon_pos = line.find(":"); - if (colon_pos == std::string::npos) continue; - - std::string id_part = line.substr(0, colon_pos); - std::string token_part = line.substr(colon_pos + 1); - - size_t id_start = id_part.find("\""); - size_t id_end = id_part.find("\"", id_start + 1); - if (id_start == std::string::npos || id_end == std::string::npos) continue; - - uint32_t token_id = - static_cast(std::stoul(id_part.substr(id_start + 1, id_end - id_start - 1))); - - size_t token_start = token_part.find("\""); - if (token_start == std::string::npos) continue; - size_t value_pos = token_start + 1; - std::string token_content = extract_json_string(token_part, value_pos); - special_tokens[token_content] = token_id; - } - } - } - } - } - - size_t slash_pos = config_file.find_last_of("/\\"); - std::string dir = (slash_pos == std::string::npos) ? "." : config_file.substr(0, slash_pos); - load_tokenizer_json_added_special_tokens(dir + "/tokenizer.json", special_tokens); -} - -std::vector split_with_special_tokens(const std::string& text, - const std::unordered_map& special_tokens) { - std::vector result; - size_t start = 0; - while (start < text.size()) { - size_t best_match_pos = text.size(); - size_t best_match_len = 0; - std::string best_special_token; - - for (const auto& [special_token, token_id] : special_tokens) { - size_t pos = text.find(special_token, start); - if (pos != std::string::npos && - (pos < best_match_pos || (pos == best_match_pos && special_token.length() > best_match_len))) { - best_match_pos = pos; - best_match_len = special_token.length(); - best_special_token = special_token; - } - } - - if (best_match_pos < text.size()) { - if (best_match_pos > start) - result.push_back(text.substr(start, best_match_pos - start)); - result.push_back(best_special_token); - start = best_match_pos + best_match_len; - } else { - if (start < text.size()) - result.push_back(text.substr(start)); - break; - } - } - return result; -} - -void Tokenizer::load_chat_template(const std::string& template_file) { - std::ifstream file(template_file); - if (!file.is_open()) { - has_chat_template_ = false; - return; - } - chat_template_ = std::string((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - has_chat_template_ = !chat_template_.empty(); -} - -void Tokenizer::detect_model_type(const std::string& config_path) { - std::ifstream file(config_path); - if (!file.is_open()) { - model_type_ = ModelType::UNKNOWN; - return; - } - - std::string line; - while (std::getline(file, line)) { - size_t pos = line.find("model_type"); - if (pos != std::string::npos) { - std::transform(line.begin(), line.end(), line.begin(), ::tolower); - - if (line.find("qwen3_5") != std::string::npos) { - model_type_ = ModelType::QWEN3P5; - break; - } else if (line.find("qwen") != std::string::npos) { - model_type_ = ModelType::QWEN; - break; - } else if (line.find("gemma4") != std::string::npos || line.find("tinyllama") != std::string::npos) { - model_type_ = ModelType::GEMMA4; - break; - } else if (line.find("gemma") != std::string::npos) { - model_type_ = ModelType::GEMMA; - break; - } else if(line.find("lfm2") != std::string::npos) { - model_type_ = ModelType::LFM2; - } else if (line.find("bert") != std::string::npos) { - model_type_ = ModelType::BERT; - break; - } else if (line.find("whisper") != std::string::npos) { - model_type_ = ModelType::WHISPER; - break; - } else if (line.find("parakeet") != std::string::npos) { - model_type_ = ModelType::PARAKEET; - break; - } else if (line.find("needle") != std::string::npos) { - model_type_ = ModelType::NEEDLE; - break; - } else if (line.find("youtu") != std::string::npos) { - model_type_ = ModelType::YOUTU; - break; - } else { - model_type_ = ModelType::UNKNOWN; - } - } - } - file.clear(); - file.seekg(0); - - while (std::getline(file, line)) { - size_t pos2 = line.find("model_variant"); - if (pos2 != std::string::npos) { - std::transform(line.begin(), line.end(), line.begin(), ::tolower); - - if (line.find("vlm") != std::string::npos) { - model_variant_ = ModelVariant::VLM; - break; - } else if (line.find("extract") != std::string::npos) { - model_variant_ = ModelVariant::EXTRACT; - break; - } else if (line.find("rag") != std::string::npos) { - model_variant_ = ModelVariant::RAG; - break; - } else { - model_variant_ = ModelVariant::DEFAULT; - } - } - } - - file.clear(); - file.seekg(0); - while (std::getline(file, line)) { - auto parse_uint = [&](const std::string& key, uint32_t& out) { - size_t p = line.find(key + "="); - if (p != std::string::npos) { - out = static_cast(std::stoul(line.substr(p + key.size() + 1))); - } - }; - parse_uint("vision_patch_size", vision_patch_size_); - parse_uint("vision_pooling_kernel_size", vision_pooling_kernel_size_); - parse_uint("vision_default_output_length", vision_default_output_length_); - parse_uint("vision_image_size", vision_image_size_); - } - - file.close(); -} - -std::string Tokenizer::get_default_stop_sequence() const { - switch (model_type_) { - case ModelType::GEMMA: - return ""; - case ModelType::GEMMA4: - return ""; - case ModelType::QWEN: - case ModelType::QWEN3P5: - case ModelType::LFM2: - return "<|im_end|>"; - case ModelType::NEEDLE: - return ""; - default: - return "<|im_end|>"; - } -} - -std::vector Tokenizer::apply_chat_template(const std::vector& messages, bool add_generation_prompt) const { - std::string formatted_prompt = format_chat_prompt(messages, add_generation_prompt); - return encode(formatted_prompt); -} - -std::string Tokenizer::format_chat_prompt(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json, bool enable_thinking_if_supported) const { - bool has_images = false; - for (const auto& msg : messages) { - if (!msg.images.empty()) { - has_images = true; - break; - } - } - if (model_type_ == ModelType::LFM2 && has_images) { - return format_lfm2_vl_style(messages, add_generation_prompt, tools_json); - } - - switch (model_type_) { - case ModelType::QWEN: - case ModelType::QWEN3P5: - return format_qwen_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); - case ModelType::GEMMA: - return format_gemma_style(messages, add_generation_prompt, tools_json); - case ModelType::GEMMA4: - return format_gemma4_style(messages, add_generation_prompt, tools_json, enable_thinking_if_supported); - case ModelType::LFM2: - return format_lfm2_style(messages, add_generation_prompt, tools_json); - case ModelType::NEEDLE: - return format_needle_style(messages, add_generation_prompt, tools_json); - case ModelType::YOUTU: - return format_youtu_style(messages, add_generation_prompt, tools_json); - default: - return format_qwen_style(messages, add_generation_prompt, tools_json); - } -} - -std::string Tokenizer::format_needle_style(const std::vector& messages, - bool /*add_generation_prompt*/, - const std::string& tools_json) const { - std::string serialized_tools = tools_json.empty() ? "[]" : tools_json; - return format_needle_query_text(messages) + "" + serialized_tools + ""; -} - -std::string Tokenizer::format_qwen_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json, bool enable_thinking_if_supported) const { - std::string result; - size_t start_idx = 0; - - if (!tools_json.empty()) { - result += "<|im_start|>system\n"; - if (!messages.empty() && messages[0].role == "system") { - result += messages[0].content + "\n\n"; - start_idx = 1; - } - result += "# Tools\n\n"; - result += "You may call one or more functions to assist with the user query.\n\n"; - result += "You are provided with function signatures within XML tags:\n"; - result += ""; - result += tools_json; - result += "\n\n\n"; - result += "For each function call, return a json object with function name and arguments within XML tags:\n"; - result += "\n"; - result += "{\"name\": , \"arguments\": }\n"; - result += ""; - result += "<|im_end|>\n"; - } else { - if (!messages.empty() && messages[0].role == "system") { - result += "<|im_start|>system\n" + messages[0].content + "<|im_end|>\n"; - start_idx = 1; - } - } - - size_t last_query_index = messages.empty() ? 0 : messages.size() - 1; - for (size_t idx = messages.size(); idx > 0; idx--) { - const auto& m = messages[idx - 1]; - if (m.role == "user") { - bool is_tool_response = m.content.size() >= 31 && - m.content.substr(0, 15) == "" && - m.content.substr(m.content.size() - 16) == ""; - if (!is_tool_response) { - last_query_index = idx - 1; - break; - } - } - } - - for (size_t i = start_idx; i < messages.size(); i++) { - const auto& msg = messages[i]; - if (msg.role == "user" || (msg.role == "system" && i > 0)) { - result += "<|im_start|>" + msg.role + "\n" + msg.content + "<|im_end|>\n"; - } else if (msg.role == "assistant") { - if (i > last_query_index && i == messages.size() - 1) { - result += "<|im_start|>assistant\n\n\n\n\n"; - std::string content = msg.content; - size_t lstrip = content.find_first_not_of('\n'); - if (lstrip != std::string::npos) { - result += content.substr(lstrip); - } - } else { - result += "<|im_start|>assistant\n" + msg.content; - } - if (!msg.tool_calls.empty()) { - for (size_t j = 0; j < msg.tool_calls.size(); j++) { - if ((j == 0 && !msg.content.empty()) || j > 0) { - result += "\n"; - } - result += "\n{\"name\": \"" + msg.tool_calls[j].name - + "\", \"arguments\": " + msg.tool_calls[j].arguments - + "}\n"; - } - } - result += "<|im_end|>\n"; - } else if (msg.role == "tool") { - bool first_tool = (i == 0 || messages[i - 1].role != "tool"); - bool last_tool = (i == messages.size() - 1 || messages[i + 1].role != "tool"); - if (first_tool) { - result += "<|im_start|>user"; - } - result += "\n\n" + msg.content + "\n"; - if (last_tool) { - result += "<|im_end|>\n"; - } - } - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - if (!enable_thinking_if_supported) { - result += "\n\n\n\n"; - } - } - - return result; -} - -std::string Tokenizer::format_lfm2_style(const std::vector& messages, - bool add_generation_prompt, - const std::string& tools_json) const -{ - std::string result = "<|startoftext|>"; - - std::string sys_content; - bool has_system_msg = false; - for (const auto& msg : messages) { - if (msg.role == "system") { - sys_content = msg.content; - has_system_msg = true; - break; - } - } - - if (!tools_json.empty()) { - if (!sys_content.empty()) { - sys_content += "\n"; - } - sys_content += "List of tools: <|tool_list_start|>["; - if (!tools_json.empty()) { - sys_content += "\n"; - sys_content += tools_json; - sys_content += "\n"; - } - sys_content += "]<|tool_list_end|>"; - sys_content += "\n\nWhen you need to call a tool, use this exact format:\n"; - sys_content += "<|tool_call_start|>[function_name(arg1=\"value1\", arg2=\"value2\")]<|tool_call_end|>\n"; - sys_content += "You can call multiple tools by using multiple tool call blocks."; - } - - if (!sys_content.empty()) { - result += "<|im_start|>system\n"; - result += sys_content; - result += "<|im_end|>\n"; - } - - for (const auto& msg : messages) { - if (msg.role == "system" && has_system_msg) { - has_system_msg = false; - continue; - } - result += "<|im_start|>" + msg.role + "\n"; - if (msg.role == "tool") { - result += "<|tool_response_start|>"; - result += msg.content; - result += "<|tool_response_end|>"; - } else { - result += msg.content; - if (msg.role == "assistant" && !msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += "<|tool_call_start|>[{\"name\": \"" + tc.name - + "\", \"arguments\": " + tc.arguments + "}]<|tool_call_end|>"; - } - } - } - result += "<|im_end|>\n"; - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - } - - return result; -} - -static std::string format_tool_call_for_prompt(const std::string& name, const std::string& args_json, bool use_pipe_tags) { - const char* tag_start = use_pipe_tags ? "<|tool_call>" : ""; - const char* tag_end = use_pipe_tags ? "" : ""; - - std::string formatted_args = "{"; - if (!args_json.empty()) { - std::map args; - size_t pos = 0; - while (pos < args_json.length() && args_json[pos] != '{') pos++; - if (pos < args_json.length()) pos++; - - while (pos < args_json.length()) { - while (pos < args_json.length() && std::isspace(static_cast(args_json[pos]))) pos++; - if (pos >= args_json.length() || args_json[pos] == '}') break; - if (args_json[pos] == ',') { pos++; continue; } - if (args_json[pos] != '"') break; - pos++; - - std::string key; - while (pos < args_json.length() && args_json[pos] != '"') { - if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { pos++; } - key += args_json[pos++]; - } - if (pos < args_json.length()) pos++; - - while (pos < args_json.length() && (std::isspace(static_cast(args_json[pos])) || args_json[pos] == ':')) pos++; - - std::string value; - if (pos < args_json.length() && args_json[pos] == '"') { - pos++; - std::string str_val; - while (pos < args_json.length() && args_json[pos] != '"') { - if (args_json[pos] == '\\' && pos + 1 < args_json.length()) { - pos++; - if (args_json[pos] == 'n') str_val += '\n'; - else if (args_json[pos] == 't') str_val += '\t'; - else str_val += args_json[pos]; - } else { - str_val += args_json[pos]; - } - pos++; - } - if (pos < args_json.length()) pos++; - value = "" + str_val + ""; - } else { - size_t val_start = pos; - while (pos < args_json.length() && args_json[pos] != ',' && args_json[pos] != '}') pos++; - value = args_json.substr(val_start, pos - val_start); - while (!value.empty() && std::isspace(static_cast(value.back()))) value.pop_back(); - } - - if (!key.empty()) args[key] = value; - } - - bool first = true; - for (const auto& [k, v] : args) { - if (!first) formatted_args += ","; - first = false; - formatted_args += k + ":" + v; - } - } - formatted_args += "}"; - - return std::string(tag_start) + "call:" + name + formatted_args + tag_end; -} - -std::string Tokenizer::format_gemma4_style(const std::vector& messages, bool add_generation_prompt, - const std::string& tools_json, bool enable_thinking_if_supported) const { - std::string result = ""; - - std::string sys_content; - size_t first_msg = 0; - if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { - sys_content = messages[0].content; - first_msg = 1; - } - - if (enable_thinking_if_supported || !sys_content.empty() || !tools_json.empty()) { - result += "<|turn>system\n"; - if (enable_thinking_if_supported) { - result += "<|think|>"; - } - result += sys_content; - result += tools_json; - result += "\n"; - } - - auto strip_channel = [](const std::string& text) -> std::string { - const std::string open_tag = "<|channel>"; - const std::string close_tag = ""; - std::string out; - size_t pos = 0; - while (pos < text.size()) { - size_t open_pos = text.find(open_tag, pos); - if (open_pos == std::string::npos) { - out += text.substr(pos); - break; - } - out += text.substr(pos, open_pos - pos); - size_t close_pos = text.find(close_tag, open_pos + open_tag.size()); - if (close_pos == std::string::npos) { - break; - } - pos = close_pos + close_tag.size(); - } - return out; - }; - - auto compute_soft_tokens = [&](const std::string& image_path) -> size_t { - int w = 0, h = 0, c = 0; - unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &c, 3); - if (!data) return 0; - stbi_image_free(data); - - uint32_t p = vision_patch_size_; - uint32_t k = vision_pooling_kernel_size_; - uint32_t side = k * p; - uint32_t max_patches = vision_default_output_length_ * k * k; - float factor = std::sqrt(static_cast(max_patches) * p * p / - (static_cast(h) * w)); - int th = static_cast(std::floor(factor * h / side)) * side; - int tw = static_cast(std::floor(factor * w / side)) * side; - if (th == 0) th = side; - if (tw == 0) tw = side; - return static_cast((th / p / k) * (tw / p / k)); - }; - - for (size_t i = first_msg; i < messages.size(); i++) { - const auto& msg = messages[i]; - std::string role = (msg.role == "assistant") ? "model" : msg.role; - result += "<|turn>" + role + "\n"; - if (role == "model") { - result += strip_channel(msg.content); - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += format_tool_call_for_prompt(tc.name, tc.arguments, true); - } - } - } else { - for (const auto& image_path : msg.images) { - size_t n = compute_soft_tokens(image_path); - if (n > 0) { - result += "\n\n<|image>"; - for (size_t j = 0; j < n; j++) - result += "<|image|>"; - result += "\n\n"; - } - } - result += msg.content; - if (msg.audio_soft_token_count > 0) { - result += "<|audio>"; - for (size_t j = 0; j < msg.audio_soft_token_count; j++) - result += "<|audio|>"; - result += ""; - } - } - result += "\n"; - } - - if (add_generation_prompt) { - result += "<|turn>model\n"; - } - - return result; -} - -std::string Tokenizer::format_lfm2_vl_style( - const std::vector& messages, - bool add_generation_prompt, - const std::string& tools_json) const -{ - if (!tools_json.empty()) { - return "ERROR: Tool calls are not supported for LFM2-VL models"; - } - - std::string result = "<|startoftext|>"; - - for (const auto& msg : messages) { - result += "<|im_start|>" + msg.role + "\n"; - for (const auto& image_path : msg.images) { - int width = 0, height = 0, channels = 0; - unsigned char* img_data = stbi_load(image_path.c_str(), &width, &height, &channels, 0); - - if (img_data) { - Siglip2Preprocessor preprocessor; - auto shape_result = preprocessor.compute_spatial_shapes(height, width); - int downsample_factor = 2; - bool use_thumbnail = true; - int grid_rows = shape_result.grid_rows; - int grid_cols = shape_result.grid_cols; - int num_tiles = grid_rows * grid_cols; - result += "<|image_start|>"; - - if (num_tiles > 1) { - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - int row = tile_idx / grid_cols; - int col = tile_idx % grid_cols; - - result += "<|img_row_" + std::to_string(row + 1) + "_col_" + std::to_string(col + 1) + "|>"; - auto [tile_height, tile_width] = shape_result.shapes[tile_idx]; - int tile_tokens = (tile_height * tile_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < tile_tokens; ++t) { - result += ""; - } - } - if (use_thumbnail && static_cast(num_tiles) < shape_result.shapes.size()) { - result += "<|img_thumbnail|>"; - - auto [thumb_height, thumb_width] = shape_result.shapes[num_tiles]; - int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < thumbnail_tokens; ++t) { - result += ""; - } - } - } else if (num_tiles == 1) { - auto [thumb_height, thumb_width] = shape_result.shapes[0]; - int thumbnail_tokens = (thumb_height * thumb_width) / (downsample_factor * downsample_factor); - - for (int t = 0; t < thumbnail_tokens; ++t) { - result += ""; - } - } - - result += "<|image_end|>"; - - stbi_image_free(img_data); - } else { - result += ""; - } - } - result += msg.content; - - result += "<|im_end|>\n"; - } - - if (add_generation_prompt) { - result += "<|im_start|>assistant\n"; - } - - return result; -} - - -std::string Tokenizer::format_gemma_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { - auto trim = [](const std::string& s) -> std::string { - size_t start = 0; - while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; - size_t end = s.size(); - while (end > start && std::isspace(static_cast(s[end - 1]))) --end; - return s.substr(start, end - start); - }; - - std::string result = ""; - - std::string system_content; - size_t start_idx = 0; - - if (!messages.empty() && (messages[0].role == "system" || messages[0].role == "developer")) { - system_content = messages[0].content; - start_idx = 1; - } - - if (!tools_json.empty() || !system_content.empty()) { - result += "developer\n"; - if (!system_content.empty()) { - result += trim(system_content); - } - if (!tools_json.empty()) { - result += tools_json; - } - result += "\n"; - } - - std::string prev_message_type; - - for (size_t i = start_idx; i < messages.size(); i++) { - const auto& msg = messages[i]; - - if (msg.role == "tool") { - std::string func_name = msg.name.empty() ? "tool" : msg.name; - result += "response:" + func_name + "{" + msg.content + "}"; - prev_message_type = "tool_response"; - - } else if (msg.role == "user") { - if (prev_message_type != "tool_response") { - result += "user\n"; - } - result += trim(msg.content); - result += "\n"; - prev_message_type = "content"; - - } else if (msg.role == "assistant" || msg.role == "model") { - if (prev_message_type != "tool_response") { - result += "model\n"; - } - if (!msg.content.empty()) { - result += trim(msg.content); - } - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += format_tool_call_for_prompt(tc.name, tc.arguments, false); - } - if (i == messages.size() - 1) { - result += ""; - } - prev_message_type = "tool_call"; - } else { - result += "\n"; - prev_message_type = "content"; - } - } - } - - if (add_generation_prompt) { - if (prev_message_type != "tool_response" && prev_message_type != "tool_call") { - result += "model\n"; - } - } - - return result; -} - -std::string Tokenizer::format_youtu_style(const std::vector& messages, bool add_generation_prompt, const std::string& tools_json) const { - std::string result = "<|begin_of_text|>"; - - std::string system_block; - for (const auto& msg : messages) { - if (msg.role == "system") { system_block = msg.content; break; } - } - - if (!tools_json.empty()) { - std::string tool_desc = - "<|begin_of_tool_description|>Tool calling capabilities.\n" - "You may call one or more functions to assist with the user query. " - "You have the following functions available:\n" - "```json\n" + tools_json + "\n```\n" - "For tool call returns, you MUST use the following format:\n" - "{\"name\": \"function-name\", \"arguments\": {\"param1\": \"value1\", \"param2\": \"value2\"}}\n" - "<|end_of_tool_description|>"; - system_block = system_block.empty() ? tool_desc : system_block + "\n\n" + tool_desc; - } - - result += system_block; - - bool is_last_user = false; - bool is_tool = false; - bool is_output_first = true; - - for (const auto& msg : messages) { - if (msg.role == "system") { - continue; - } else if (msg.role == "user") { - is_last_user = true; is_tool = false; - result += "<|User|>" + msg.content; - } else if (msg.role == "tool") { - is_tool = true; is_last_user = false; - if (is_output_first) { - result += "<|User|>" + msg.content + ""; - is_output_first = false; - } else { - result += "\n" + msg.content + ""; - } - } else if (msg.role == "assistant" || msg.role == "model") { - is_last_user = false; is_tool = false; is_output_first = true; - result += "<|Assistant|>" + msg.content; - if (!msg.tool_calls.empty()) { - for (const auto& tc : msg.tool_calls) { - result += "{\"name\": \"" + tc.name - + "\", \"arguments\": " + tc.arguments + "}"; - } - } - result += "<|end_of_text|>"; - } - } - - if (add_generation_prompt && (is_last_user || is_tool)) { - result += "<|Assistant|>"; - } - - return result; -} - -} // namespace engine -} // namespace cactus diff --git a/cactus/ffi/cactus_cloud.cpp b/cactus/ffi/cactus_cloud.cpp deleted file mode 100644 index 93114cefc..000000000 --- a/cactus/ffi/cactus_cloud.cpp +++ /dev/null @@ -1,462 +0,0 @@ -#include "cactus_cloud.h" -#include "telemetry/telemetry.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef CACTUS_USE_CURL -#include -#endif - -namespace cactus { -namespace ffi { - -namespace { - -std::string compact_error_detail(std::string detail) { - detail = trim_string(detail); - if (detail.empty()) return {}; - - for (char& c : detail) { - if (c == '\n' || c == '\r' || c == '\t') c = ' '; - } - while (detail.find(" ") != std::string::npos) { - detail.erase(detail.find(" "), 1); - } - - constexpr size_t kMaxLen = 160; - if (detail.size() > kMaxLen) { - detail = detail.substr(0, kMaxLen) + "..."; - } - return detail; -} - -} // namespace - -std::string resolve_cloud_api_key(const char* cloud_key_param) { - const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); - const char* env_cloud_legacy = std::getenv("CACTUS_CLOUD_API_KEY"); // keeping this cuz Matthew Hayes had some code reliant on Cactus_cloud_api_key - std::string resolved_cloud_key; - bool key_from_param_or_env = false; - - if (cloud_key_param && *cloud_key_param) { - resolved_cloud_key = cloud_key_param; - key_from_param_or_env = true; - } else if (env_cloud && *env_cloud) { - resolved_cloud_key = env_cloud; - key_from_param_or_env = true; - } else if (env_cloud_legacy && *env_cloud_legacy) { - resolved_cloud_key = env_cloud_legacy; - key_from_param_or_env = true; - } else { - resolved_cloud_key = cactus::telemetry::loadCachedCloudApiKey(); - } - - resolved_cloud_key = trim_string(resolved_cloud_key); - - const bool should_cache = - !resolved_cloud_key.empty() && - key_from_param_or_env; - if (should_cache) { - cactus::telemetry::cacheCloudApiKey(resolved_cloud_key.c_str()); - } - - return resolved_cloud_key; -} - -namespace { - -#ifdef CACTUS_USE_CURL -static std::atomic g_warned_missing_cloud_api_key{false}; - -static void apply_curl_tls_trust(CURL* curl) { - if (!curl) return; - const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); - if (ca_bundle && ca_bundle[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); - } -#if defined(__ANDROID__) - const char* ca_path = std::getenv("CACTUS_CA_PATH"); - if (ca_path && ca_path[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); - } else { - curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); - } -#endif -} - -static size_t curl_write_cb(void* ptr, size_t size, size_t nmemb, void* userdata) { - auto* s = static_cast(userdata); - s->append(static_cast(ptr), size * nmemb); - return size * nmemb; -} - -static bool read_file_bytes(const std::string& path, std::vector& out) { - std::ifstream in(path, std::ios::binary); - if (!in.is_open()) return false; - in.seekg(0, std::ios::end); - std::streamsize size = in.tellg(); - if (size <= 0) return false; - in.seekg(0, std::ios::beg); - - out.resize(static_cast(size)); - if (!in.read(reinterpret_cast(out.data()), size)) return false; - return true; -} - -static std::string infer_mime_type(const std::string& path) { - auto dot = path.find_last_of('.'); - if (dot == std::string::npos) return "image/jpeg"; - std::string ext = path.substr(dot + 1); - std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - - if (ext == "png") return "image/png"; - if (ext == "webp") return "image/webp"; - if (ext == "gif") return "image/gif"; - return "image/jpeg"; -} - -static std::string build_cloud_text_prompt(const CloudCompletionRequest& request) { - std::ostringstream oss; - oss << "You are continuing an existing assistant conversation.\\n"; - oss << "Output contract:\\n"; - oss << "1) Never include role prefixes like 'assistant:'.\\n"; - oss << "2) Never include markdown/code fences/backticks.\\n"; - oss << "3) Return only the final assistant answer text unless a tool call is required.\\n"; - oss << "4) If a tool call is required, return ONLY JSON with this exact shape:\\n"; - oss << "[{\"name\":\"tool_name\",\"arguments\":{\"arg\":\"value\"}}]\\n"; - oss << "5) Do not include any prose before or after that JSON tool-call output.\\n"; - oss << "\\nConversation:\\n"; - for (const auto& m : request.messages) { - oss << "[" << m.role << "] " << m.content << "\\n"; - } - - if (!request.tools.empty()) { - oss << "\\nAvailable tools JSON (use only these tool names and arguments):\\n"; - oss << serialize_tools_json(request.tools) << "\\n"; - oss << "If tools are relevant, prefer the strict JSON tool-call output contract above.\\n"; - } - - if (!request.local_output.empty()) { - oss << "\\nLocal model draft (useful fallback reference, may be low confidence):\\n"; - oss << request.local_output << "\\n"; - } - - return oss.str(); -} - -static std::string call_cloud_endpoint(const std::string& url, - const std::string& payload, - long timeout_ms, - const char* cloud_key_param, - std::string& err_out) { - std::string api_key = resolve_cloud_api_key(cloud_key_param); - if (api_key.empty()) { - if (!g_warned_missing_cloud_api_key.exchange(true)) { - CACTUS_LOG_WARN("cloud_handoff", "No cloud key found (cloud_key param, CACTUS_CLOUD_KEY, or CACTUS_CLOUD_API_KEY env); cloud handoff will fall back to local output"); - } - err_out = "missing_api_key"; - return {}; - } - - CURL* curl = curl_easy_init(); - if (!curl) { - err_out = "curl_init_failed"; - return {}; - } - - std::string response_body; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, ("X-API-Key: " + api_key).c_str()); - headers = curl_slist_append(headers, "Content-Type: application/json"); - - const char* extra_hdrs = std::getenv("CACTUS_CLOUD_HEADERS"); - if (extra_hdrs && extra_hdrs[0] != '\0') { - std::istringstream stream(extra_hdrs); - std::string pair; - while (std::getline(stream, pair, ',')) { - auto eq = pair.find('='); - if (eq != std::string::npos && eq > 0) { - std::string header = pair.substr(0, eq) + ": " + pair.substr(eq + 1); - headers = curl_slist_append(headers, header.c_str()); - } - } - } - - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, static_cast(payload.size())); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_body); - curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, timeout_ms); - - if (!env_flag_enabled("CACTUS_CLOUD_STRICT_SSL")) { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); - } else { - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - apply_curl_tls_trust(curl); - } - - CURLcode res = curl_easy_perform(curl); - long http_status = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_status); - curl_slist_free_all(headers); - curl_easy_cleanup(curl); - - if (res != CURLE_OK) { - err_out = curl_easy_strerror(res); - return {}; - } - - if (http_status >= 400) { - std::string detail = json_string_field(response_body, "error"); - if (detail.empty()) detail = json_string_field(response_body, "message"); - if (detail.empty()) detail = json_string_field(response_body, "detail"); - detail = compact_error_detail(detail); - - err_out = "http_" + std::to_string(http_status); - if (!detail.empty()) { - err_out += ":" + detail; - } - return {}; - } - - return response_body; -} -#endif - -} // namespace - -std::string cloud_base64_encode(const uint8_t* data, size_t len) { - static const char table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - std::string out; - out.reserve(((len + 2) / 3) * 4); - for (size_t i = 0; i < len; i += 3) { - uint32_t n = static_cast(data[i]) << 16; - if (i + 1 < len) n |= static_cast(data[i + 1]) << 8; - if (i + 2 < len) n |= static_cast(data[i + 2]); - out += table[(n >> 18) & 0x3F]; - out += table[(n >> 12) & 0x3F]; - out += (i + 1 < len) ? table[(n >> 6) & 0x3F] : '='; - out += (i + 2 < len) ? table[n & 0x3F] : '='; - } - return out; -} - -std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes) { - constexpr uint32_t sample_rate = 16000; - constexpr uint16_t channels = 1; - constexpr uint16_t bits = 16; - const uint32_t byte_rate = sample_rate * channels * bits / 8; - const uint16_t block_align = channels * bits / 8; - const uint32_t data_size = static_cast(pcm_bytes); - const uint32_t file_size = 36 + data_size; - - std::vector wav(44 + pcm_bytes); - auto w16 = [&](size_t off, uint16_t v) { - wav[off] = v & 0xFF; - wav[off + 1] = v >> 8; - }; - auto w32 = [&](size_t off, uint32_t v) { - wav[off] = v & 0xFF; - wav[off + 1] = (v >> 8) & 0xFF; - wav[off + 2] = (v >> 16) & 0xFF; - wav[off + 3] = (v >> 24) & 0xFF; - }; - - std::memcpy(wav.data(), "RIFF", 4); - w32(4, file_size); - std::memcpy(wav.data() + 8, "WAVE", 4); - std::memcpy(wav.data() + 12, "fmt ", 4); - w32(16, 16); - w16(20, 1); - w16(22, channels); - w32(24, sample_rate); - w32(28, byte_rate); - w16(32, block_align); - w16(34, bits); - std::memcpy(wav.data() + 36, "data", 4); - w32(40, data_size); - std::memcpy(wav.data() + 44, pcm, pcm_bytes); - return wav; -} - -CloudResponse cloud_transcribe_request(const std::string& audio_b64, - const std::string& fallback_text, - long timeout_seconds, - const char* cloud_key) { -#ifdef CACTUS_USE_CURL - std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); - std::string endpoint = base + "/transcribe"; - - std::string payload = "{\"audio\":\"" + audio_b64 + "\",\"mime_type\":\"audio/wav\",\"language\":\"en-US\"}"; - - std::string err; - std::string body = call_cloud_endpoint(endpoint, payload, timeout_seconds * 1000L, cloud_key, err); - if (body.empty()) { - return {fallback_text, "", false, err.empty() ? "request_failed" : err}; - } - - std::string transcript = json_string_field(body, "transcript"); - if (transcript.empty()) { - transcript = json_string_field(body, "text"); - } - if (transcript.empty()) { - transcript = json_string_field(body, "response"); - } - if (transcript.empty()) { - transcript = json_string_field(body, "analysis"); - } - if (transcript.empty()) { - std::string detail = json_string_field(body, "error"); - if (detail.empty()) detail = json_string_field(body, "message"); - if (detail.empty()) detail = json_string_field(body, "detail"); - detail = compact_error_detail(detail); - if (!detail.empty()) { - return {fallback_text, "", false, "missing_transcript:" + detail}; - } - return {fallback_text, "", false, "missing_transcript"}; - } - - std::string api_key_hash = json_string_field(body, "api_key_hash"); - return {transcript, api_key_hash, true, ""}; -#else - (void)audio_b64; - (void)timeout_seconds; - return {fallback_text, "", false, "curl_not_enabled"}; -#endif -} - -CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, - long timeout_ms) { -#ifdef CACTUS_USE_CURL - std::string base = env_or_default("CACTUS_CLOUD_API_BASE", "https://104.198.76.3/api/v1"); - std::string model = env_or_default("CACTUS_CLOUD_MODEL", "gemini-2.5-flash"); - - std::string endpoint; - std::string payload; - - std::string img_b64; - std::string img_mime; - if (request.has_images) { - for (const auto& message : request.messages) { - for (const auto& image : message.images) { - if (!image.empty()) { - std::vector img_bytes; - if (read_file_bytes(image, img_bytes)) { - img_b64 = cloud_base64_encode(img_bytes.data(), img_bytes.size()); - img_mime = infer_mime_type(image); - } - break; - } - } - if (!img_b64.empty()) break; - } - } - - std::string audio_b64; - if (request.has_audio && !request.audio_pcm.empty()) { - auto wav = cloud_build_wav(request.audio_pcm.data(), request.audio_pcm.size()); - audio_b64 = cloud_base64_encode(wav.data(), wav.size()); - } - - std::string text_prompt = build_cloud_text_prompt(request); - bool use_omni = !audio_b64.empty() || (request.has_images && request.has_audio); - - if (use_omni) { - endpoint = base + "/omni"; - payload = "{"; - payload += "\"text\":\"" + escape_json_string(text_prompt) + "\""; - if (!img_b64.empty()) { - payload += ",\"image\":\"" + img_b64 + "\""; - payload += ",\"image_mime_type\":\"" + img_mime + "\""; - } - if (!audio_b64.empty()) { - payload += ",\"audio\":\"" + audio_b64 + "\""; - payload += ",\"audio_mime_type\":\"audio/wav\""; - } - payload += ",\"model\":\"" + escape_json_string(model) + "\""; - payload += ",\"language\":\"en-US\""; - payload += "}"; - } else if (request.has_images && !img_b64.empty()) { - endpoint = base + "/vlm"; - payload = "{" - "\"image\":\"" + img_b64 + "\"," - "\"mime_type\":\"" + img_mime + "\"," - "\"prompt\":\"" + escape_json_string(text_prompt) + "\"," - "\"language\":\"en-US\"," - "\"model\":\"" + escape_json_string(model) + "\"" - "}"; - } else { - endpoint = base + "/text"; - payload = "{" - "\"text\":\"" + escape_json_string(text_prompt) + "\"," - "\"language\":\"en-US\"," - "\"model\":\"" + escape_json_string(model) + "\"" - "}"; - } - - std::string err; - const char* cloud_key = request.cloud_key.empty() ? nullptr : request.cloud_key.c_str(); - std::string body = call_cloud_endpoint(endpoint, payload, timeout_ms, cloud_key, err); - if (body.empty()) { - return {false, false, "", {}, err.empty() ? "request_failed" : err}; - } - - std::vector function_calls; - std::string calls_json = json_array_field(body, "function_calls"); - if (calls_json != "[]") { - function_calls = split_json_array(calls_json); - } - - std::string response = json_string_field(body, "text"); - if (response.empty()) { - response = json_string_field(body, "analysis"); - } - - if (!response.empty() && function_calls.empty()) { - std::string regular_response; - parse_function_calls_from_response(response, regular_response, function_calls); - response = regular_response; - } - - if (!response.empty() && function_calls.empty()) { - auto first = response.find_first_not_of(" \t\n\r"); - auto last = response.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) { - std::string trimmed = response.substr(first, last - first + 1); - if (!trimmed.empty() && trimmed.front() == '[' && trimmed.back() == ']' && - trimmed.find("\"name\"") != std::string::npos) { - function_calls = split_json_array(trimmed); - response.clear(); - } - } - } - - if (response.empty() && function_calls.empty()) { - return {false, false, "", {}, "missing_text"}; - } - - CloudCompletionResult out; - out.ok = true; - out.used_cloud = true; - out.response = response; - out.function_calls = std::move(function_calls); - return out; -#else - (void)request; - (void)timeout_ms; - return {false, false, "", {}, "curl_not_enabled"}; -#endif -} - -} // namespace ffi -} // namespace cactus diff --git a/cactus/ffi/cactus_cloud.h b/cactus/ffi/cactus_cloud.h deleted file mode 100644 index 6911d9a91..000000000 --- a/cactus/ffi/cactus_cloud.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef CACTUS_CLOUD_H -#define CACTUS_CLOUD_H - -#include "cactus_utils.h" -#include -#include - -namespace cactus { -namespace ffi { - -struct CloudResponse { - std::string transcript; - std::string api_key_hash; - bool used_cloud = false; - std::string error; -}; - -struct CloudCompletionRequest { - std::vector messages; - std::vector tools; - std::string local_output; - std::vector local_function_calls; - bool has_images = false; - bool has_audio = false; - std::vector audio_pcm; - std::string cloud_key; -}; - -struct CloudCompletionResult { - bool ok = false; - bool used_cloud = false; - std::string response; - std::vector function_calls; - std::string error; -}; - -std::string cloud_base64_encode(const uint8_t* data, size_t len); -std::vector cloud_build_wav(const uint8_t* pcm, size_t pcm_bytes); -std::string resolve_cloud_api_key(const char* cloud_key_param); -CloudResponse cloud_transcribe_request(const std::string& audio_b64, - const std::string& fallback_text, - long timeout_seconds = 15L, - const char* cloud_key = nullptr); -CloudCompletionResult cloud_complete_request(const CloudCompletionRequest& request, - long timeout_ms); - -} // namespace ffi -} // namespace cactus - -#endif // CACTUS_CLOUD_H diff --git a/cactus/ffi/cactus_complete.cpp b/cactus/ffi/cactus_complete.cpp deleted file mode 100644 index 461e8b8ff..000000000 --- a/cactus/ffi/cactus_complete.cpp +++ /dev/null @@ -1,1371 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_cloud.h" -#include "cactus_utils.h" -#include "needle_tools.h" -#include "telemetry/telemetry.h" -#include "../../libs/audio/wav.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t DEFAULT_ROLLING_ENTROPY_WINDOW = 10; - -namespace { - -std::vector> extract_schema_property_types(const std::string& schema); -std::vector extract_schema_required(const std::string& schema); - -std::string extract_last_user_query(const std::vector& messages) { - for (auto it = messages.rbegin(); it != messages.rend(); ++it) { - if (it->role == "user") { - return it->content; - } - } - return {}; -} - -std::vector encode_needle_tools_suffix(Tokenizer* tokenizer, const std::string& formatted_tools) { - auto prefix = tokenizer->encode(""); - auto body = tokenizer->encode(formatted_tools); - std::vector tokens; - tokens.reserve(prefix.size() + body.size() + 1); - tokens.insert(tokens.end(), prefix.begin(), prefix.end()); - tokens.insert(tokens.end(), body.begin(), body.end()); - tokens.push_back(tokenizer->get_eos_token()); - return tokens; -} - -void inject_rag_context(CactusModelHandle* handle, std::vector& messages) { - if (!handle->corpus_index) return; - - std::string query = extract_last_user_query(messages); - if (query.empty()) return; - - std::string rag_context = retrieve_rag_context(handle, query); - if (rag_context.empty()) return; - - if (!messages.empty() && messages[0].role == "system") { - messages[0].content = rag_context + messages[0].content; - } else { - ChatMessage system_msg; - system_msg.role = "system"; - system_msg.content = rag_context + "Answer the user's question using ONLY the context above. Do not use any prior knowledge. If the answer cannot be found in the context, respond with \"I don't have enough information to answer that.\""; - messages.insert(messages.begin(), system_msg); - } -} - -std::vector build_tool_constraint_specs(const std::vector& tools) { - std::vector specs; - specs.reserve(tools.size()); - - for (const auto& tool : tools) { - ToolConstraintSpec spec; - spec.name = tool.name; - - auto schema_it = tool.parameters.find("schema"); - if (schema_it != tool.parameters.end()) { - auto properties = extract_schema_property_types(schema_it->second); - spec.parameter_names.reserve(properties.size()); - for (const auto& [name, _] : properties) { - spec.parameter_names.push_back(name); - } - spec.required_parameter_names = extract_schema_required(schema_it->second); - } - - specs.push_back(std::move(spec)); - } - - return specs; -} - -void strip_thinking_from_cache(CactusModelHandle* handle, - const std::vector& generated_tokens, - size_t prompt_len) { - const auto& cfg = handle->model->get_config(); - uint32_t open_id = cfg.channel_open_token_id; - uint32_t close_id = cfg.channel_close_token_id; - auto ranges = find_channel_token_ranges(generated_tokens, prompt_len, - open_id, close_id); - if (ranges.empty()) return; - - handle->model->remove_thinking_tokens(ranges); - for (auto it = ranges.rbegin(); it != ranges.rend(); ++it) { - auto start = handle->processed_tokens.begin() + it->first; - handle->processed_tokens.erase(start, start + it->second); - } -} - -void setup_tool_constraints(CactusModelHandle* handle, const std::vector& tools, - bool force_tools, float& temperature, - const std::unordered_map& needle_name_map = {}) { - if (!force_tools || tools.empty()) return; - - auto specs = build_tool_constraint_specs(tools); - - if (!needle_name_map.empty()) { - for (auto& spec : specs) { - std::string snake = needle::to_snake_case(spec.name); - spec.name = snake; - } - } - - handle->model->set_tool_constraints(specs); - - if (temperature == 0.0f) { - temperature = 0.01f; - } -} - -size_t find_json_block_end(const std::string& json, size_t start) { - if (start >= json.size() || json[start] != '{') { - return std::string::npos; - } - - int depth = 1; - bool in_string = false; - bool escaped = false; - size_t pos = start + 1; - while (pos < json.size() && depth > 0) { - char c = json[pos]; - if (in_string) { - if (escaped) { - escaped = false; - } else if (c == '\\') { - escaped = true; - } else if (c == '"') { - in_string = false; - } - } else { - if (c == '"') { - in_string = true; - } else if (c == '{') { - depth++; - } else if (c == '}') { - depth--; - } - } - ++pos; - } - - return depth == 0 ? pos : std::string::npos; -} - -std::string extract_json_object_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t key_pos = json.find(pattern); - if (key_pos == std::string::npos) { - return {}; - } - - size_t object_start = json.find('{', key_pos + pattern.size()); - if (object_start == std::string::npos) { - return {}; - } - - size_t object_end = find_json_block_end(json, object_start); - if (object_end == std::string::npos) { - return {}; - } - - return json.substr(object_start, object_end - object_start); -} - -std::vector> extract_schema_property_types(const std::string& schema) { - std::vector> properties; - std::string properties_object = extract_json_object_field(schema, "properties"); - if (properties_object.empty() || properties_object.size() < 2) { - return properties; - } - - size_t pos = 1; - while (pos + 1 < properties_object.size()) { - size_t key_start = properties_object.find('"', pos); - if (key_start == std::string::npos || key_start + 1 >= properties_object.size()) { - break; - } - size_t key_end = properties_object.find('"', key_start + 1); - if (key_end == std::string::npos) { - break; - } - - std::string name = properties_object.substr(key_start + 1, key_end - key_start - 1); - size_t value_start = properties_object.find('{', key_end); - if (value_start == std::string::npos) { - break; - } - size_t value_end = find_json_block_end(properties_object, value_start); - if (value_end == std::string::npos) { - break; - } - - std::string value = properties_object.substr(value_start, value_end - value_start); - std::string type = "string"; - std::string type_pattern = "\"type\":\""; - size_t type_pos = value.find(type_pattern); - if (type_pos != std::string::npos) { - size_t type_start = type_pos + type_pattern.size(); - size_t type_end = value.find('"', type_start); - if (type_end != std::string::npos) { - type = value.substr(type_start, type_end - type_start); - } - } else if (value.find("\"enum\"") != std::string::npos) { - type = "string"; - } else if (value.find("\"properties\"") != std::string::npos) { - type = "object"; - } - - properties.emplace_back(std::move(name), std::move(type)); - pos = value_end; - } - - return properties; -} - -std::vector extract_schema_required(const std::string& schema) { - std::vector required; - std::string key = "\"required\""; - size_t key_pos = schema.find(key); - if (key_pos == std::string::npos) return required; - size_t arr_start = schema.find('[', key_pos + key.size()); - if (arr_start == std::string::npos) return required; - size_t arr_end = schema.find(']', arr_start); - if (arr_end == std::string::npos) return required; - size_t pos = arr_start + 1; - while (pos < arr_end) { - size_t qs = schema.find('"', pos); - if (qs == std::string::npos || qs >= arr_end) break; - size_t qe = schema.find('"', qs + 1); - if (qe == std::string::npos || qe > arr_end) break; - required.push_back(schema.substr(qs + 1, qe - qs - 1)); - pos = qe + 1; - } - return required; -} - -std::string serialize_needle_tools(const std::vector& tools, - const char* raw_tools_json, - std::unordered_map& name_map) { - name_map.clear(); - - if (raw_tools_json && std::strlen(raw_tools_json) > 2) { - std::string raw(raw_tools_json); - - bool is_needle_format = (raw.find("\"parameters\"") != std::string::npos && - raw.find("\"schema\"") == std::string::npos && - raw.find("\"type\":\"function\"") == std::string::npos); - - if (is_needle_format) { - std::string result; - result.reserve(raw.size()); - const std::string name_key = "\"name\":"; - size_t pos = 0; - - while (pos < raw.size()) { - size_t found = raw.find(name_key, pos); - if (found == std::string::npos) { - result.append(raw, pos, raw.size() - pos); - break; - } - result.append(raw, pos, found + name_key.size() - pos); - pos = found + name_key.size(); - - while (pos < raw.size() && (raw[pos] == ' ' || raw[pos] == '\t' || - raw[pos] == '\n' || raw[pos] == '\r')) { - result += raw[pos]; - pos++; - } - if (pos >= raw.size() || raw[pos] != '"') continue; - result += '"'; - pos++; - - size_t name_start = pos; - bool escaped = false; - while (pos < raw.size()) { - if (escaped) { escaped = false; pos++; continue; } - if (raw[pos] == '\\') { escaped = true; pos++; continue; } - if (raw[pos] == '"') break; - pos++; - } - std::string orig_name = raw.substr(name_start, pos - name_start); - std::string snake_name = needle::to_snake_case(orig_name); - name_map[snake_name] = orig_name; - - result += snake_name; - } - - std::string compact; - compact.reserve(result.size()); - bool in_str = false; - bool esc = false; - size_t ri = 0; - while (ri < result.size()) { - unsigned char ch = static_cast(result[ri]); - if (in_str) { - if (esc) { - compact += static_cast(ch); - esc = false; - } else if (ch == '\\') { - compact += '\\'; - esc = true; - } else if (ch == '"') { - compact += '"'; - in_str = false; - } else if (ch >= 0x80) { - uint32_t cp = 0; - int extra = 0; - if ((ch & 0xE0) == 0xC0) { cp = ch & 0x1F; extra = 1; } - else if ((ch & 0xF0) == 0xE0) { cp = ch & 0x0F; extra = 2; } - else if ((ch & 0xF8) == 0xF0) { cp = ch & 0x07; extra = 3; } - else { compact += static_cast(ch); ri++; continue; } - for (int e = 0; e < extra && ri + 1 < result.size(); e++) { - ri++; - cp = (cp << 6) | (static_cast(result[ri]) & 0x3F); - } - if (cp <= 0xFFFF) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", cp); - compact += buf; - } else { - cp -= 0x10000; - char buf[14]; - std::snprintf(buf, sizeof(buf), "\\u%04x\\u%04x", - 0xD800 + (cp >> 10), 0xDC00 + (cp & 0x3FF)); - compact += buf; - } - } else { - compact += static_cast(ch); - } - } else { - if (ch == '"') { in_str = true; compact += '"'; } - else if (ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r') { - compact += static_cast(ch); - } - } - ri++; - } - - return compact; - } - } - - if (tools.empty()) { - return "[]"; - } - - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < tools.size(); ++i) { - if (i > 0) { - oss << ","; - } - - std::string snake_name = needle::to_snake_case(tools[i].name); - name_map[snake_name] = tools[i].name; - - oss << "{\"name\":\"" << escape_json_string(snake_name) << "\""; - if (!tools[i].description.empty()) { - oss << ",\"description\":\"" << escape_json_string(tools[i].description) << "\""; - } - - oss << ",\"parameters\":"; - auto schema_it = tools[i].parameters.find("schema"); - if (schema_it == tools[i].parameters.end()) { - oss << "{}"; - } else { - auto properties = extract_schema_property_types(schema_it->second); - auto required = extract_schema_required(schema_it->second); - std::unordered_set required_set(required.begin(), required.end()); - - if (properties.empty()) { - oss << "{}"; - } else { - std::string properties_object = extract_json_object_field(schema_it->second, "properties"); - - oss << "{"; - for (size_t p = 0; p < properties.size(); ++p) { - if (p > 0) { - oss << ","; - } - const std::string& param_name = properties[p].first; - const std::string& param_type = properties[p].second; - bool is_required = required_set.count(param_name) > 0; - - std::string description; - std::string prop_obj = extract_json_object_field(properties_object, param_name); - if (!prop_obj.empty()) { - std::string desc_pattern = "\"description\":\""; - size_t desc_pos = prop_obj.find(desc_pattern); - if (desc_pos != std::string::npos) { - size_t desc_start = desc_pos + desc_pattern.size(); - size_t desc_end = desc_start; - bool escaped = false; - while (desc_end < prop_obj.size()) { - if (escaped) { escaped = false; desc_end++; continue; } - if (prop_obj[desc_end] == '\\') { escaped = true; desc_end++; continue; } - if (prop_obj[desc_end] == '"') break; - desc_end++; - } - description = prop_obj.substr(desc_start, desc_end - desc_start); - } - } - - oss << "\"" << escape_json_string(param_name) << "\":{"; - oss << "\"type\":\"" << escape_json_string(param_type) << "\""; - if (!description.empty()) { - oss << ",\"description\":\"" << description << "\""; - } - oss << ",\"required\":" << (is_required ? "true" : "false"); - oss << "}"; - } - oss << "}"; - } - } - - oss << "}"; - } - oss << "]"; - return oss.str(); -} - -std::vector> build_stop_sequences( - Tokenizer* tokenizer, - const std::vector& stop_sequences, - Config::ModelType model_type, - bool has_tools -) { - std::vector> stop_token_sequences; - stop_token_sequences.push_back({tokenizer->get_eos_token()}); - - std::vector sequences = stop_sequences; - if (sequences.empty()) { - std::string default_stop = tokenizer->get_default_stop_sequence(); - if (!default_stop.empty()) { - sequences.push_back(default_stop); - } - } - for (const auto& stop_seq : sequences) { - stop_token_sequences.push_back(tokenizer->encode(stop_seq)); - } - - if ((model_type == Config::ModelType::GEMMA || model_type == Config::ModelType::GEMMA3N) && has_tools) { - stop_token_sequences.push_back(tokenizer->encode("")); - stop_token_sequences.push_back(tokenizer->encode("")); - } - - if (model_type == Config::ModelType::GEMMA4) { - stop_token_sequences.push_back(tokenizer->encode("")); - if (has_tools) { - stop_token_sequences.push_back(tokenizer->encode("")); - stop_token_sequences.push_back(tokenizer->encode("<|tool_response>")); - } - } - - return stop_token_sequences; -} - -void trim_stop_suffix(std::vector& generated_tokens, - const std::vector>& stop_token_sequences, - bool include_stop_sequences) { - if (include_stop_sequences) return; - for (const auto& stop_seq : stop_token_sequences) { - if (stop_seq.empty()) continue; - if (generated_tokens.size() >= stop_seq.size() && - std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) { - generated_tokens.resize(generated_tokens.size() - stop_seq.size()); - break; - } - } -} - -void reset_cache(CactusModelHandle* handle) { - handle->model->reset_cache(); - handle->processed_tokens.clear(); - handle->processed_images.clear(); - handle->user_audio_counts.clear(); -} - -struct PrefillResult { - std::vector remaining_tokens; - size_t prefilled_count = 0; - bool was_prefix = false; - bool was_exact_match = false; -}; - -struct EntropyState { - std::vector window; - float window_sum = 0.0f; - float total_sum = 0.0f; - size_t total_count = 0; - bool spike_handoff = false; - size_t window_size = DEFAULT_ROLLING_ENTROPY_WINDOW; - - void add(float entropy) { - window.push_back(entropy); - window_sum += entropy; - total_sum += entropy; - total_count++; - - if (window.size() > window_size) { - window_sum -= window.front(); - window.erase(window.begin()); - } - } - - float rolling_confidence() const { - return 1.0f - (window_sum / window.size()); - } - - float mean_confidence() const { - return 1.0f - (total_sum / static_cast(total_count)); - } -}; - -struct PreparedPrompt { - InferenceOptions options; - Config::ModelType model_type = Config::ModelType::QWEN; - std::vector image_paths; - std::vector audio_paths; - std::vector messages; - std::vector tools; - std::vector tokens; - size_t context_token_count = 0; - std::vector> images; - - std::vector audio_features; - size_t audio_num_frames = 0; - - std::unordered_map needle_name_map; - - bool has_images() const { - return std::any_of(images.begin(), images.end(), - [](const auto& msg_imgs) { return !msg_imgs.empty(); }); - } - - bool has_audio() const { - return !audio_features.empty(); - } -}; - -CactusModelHandle::ProcessedImage image_signature(const std::string& image_path) { - std::filesystem::path normalized_path(image_path); - std::error_code ec; - - auto absolute_path = std::filesystem::absolute(normalized_path, ec); - if (!ec) { - normalized_path = absolute_path; - } - - CactusModelHandle::ProcessedImage image; - image.path = normalized_path.string(); - - ec.clear(); - auto status = std::filesystem::status(normalized_path, ec); - if (!ec && std::filesystem::is_regular_file(status)) { - std::error_code time_ec; - auto mtime = std::filesystem::last_write_time(normalized_path, time_ec); - if (!time_ec) { - image.last_modified_timestamp = static_cast(mtime.time_since_epoch().count()); - } - } - - return image; -} - -std::vector> images_from_message(const std::vector& messages) { - std::vector> message_signatures; - message_signatures.reserve(messages.size()); - - for (const auto& message : messages) { - std::vector image_signatures; - image_signatures.reserve(message.images.size()); - for (const auto& image_path : message.images) { - image_signatures.push_back(image_signature(image_path)); - } - message_signatures.push_back(std::move(image_signatures)); - } - - return message_signatures; -} - -bool image_context_prefix_matches( - const std::vector>& prefix, - const std::vector>& full -) { - return prefix.size() <= full.size() && - std::equal(prefix.begin(), prefix.end(), full.begin()); -} - -bool prompt_context_matches( - const CactusModelHandle* handle, - const PreparedPrompt& prompt -) { - if (handle->processed_tokens.empty()) { - return false; - } - if (handle->model && - handle->model->get_config().model_type == Config::ModelType::NEEDLE && - prompt.tokens.size() != handle->processed_tokens.size() + 1) { - return false; - } - if (prompt.context_token_count < handle->processed_tokens.size()) { - return false; - } - if (!std::equal(handle->processed_tokens.begin(), handle->processed_tokens.end(), prompt.tokens.begin())) { - return false; - } - if (prompt.has_images()) { - return image_context_prefix_matches(handle->processed_images, prompt.images); - } - return !prompt.has_images(); -} - -PreparedPrompt prepare_prompt( - CactusModelHandle* handle, - const char* messages_json, - const char* options_json, - const char* tools_json, - bool apply_tool_constraints, - bool add_generation_prompt, - const uint8_t* pcm_buffer = nullptr, - size_t pcm_buffer_size = 0 -) { - if (!handle || !handle->model) { - throw std::runtime_error("Invalid model handle"); - } - - PreparedPrompt prompt; - prompt.options = parse_inference_options_json(options_json ? options_json : ""); - prompt.messages = parse_messages_json(messages_json, prompt.image_paths, &prompt.audio_paths); - if (prompt.messages.empty()) { - throw std::runtime_error("No messages provided"); - } - - inject_rag_context(handle, prompt.messages); - - if (tools_json && std::strlen(tools_json) > 0) { - prompt.tools = parse_tools_json(tools_json); - } - - if (prompt.options.tool_rag_top_k > 0 && prompt.tools.size() > prompt.options.tool_rag_top_k) { - std::string query = extract_last_user_query(prompt.messages); - if (!query.empty()) { - prompt.tools = select_relevant_tools(handle, query, prompt.tools, prompt.options.tool_rag_top_k); - } - } - - if (prompt.model_type == Config::ModelType::NEEDLE && !prompt.tools.empty()) { - prompt.options.force_tools = true; - } - - if (apply_tool_constraints && prompt.model_type != Config::ModelType::NEEDLE) { - setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, prompt.options.temperature); - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer unavailable"); - } - - prompt.model_type = handle->model->get_config().model_type; - - if (prompt.options.confidence_threshold < 0.0f) { - float model_default = handle->model->get_config().default_cloud_handoff_threshold; - prompt.options.confidence_threshold = (model_default > 0.0f) ? model_default : 0.7f; - } - - if (prompt.model_type == Config::ModelType::GEMMA4) { - std::vector audio_samples; - if (pcm_buffer != nullptr && pcm_buffer_size > 1) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, 16000); - } else if (!prompt.audio_paths.empty()) { - for (auto it = prompt.messages.rbegin(); it != prompt.messages.rend(); ++it) { - if (!it->audio.empty()) { - const std::string& audio_path = it->audio.back(); - AudioFP32 wav = load_wav(audio_path); - audio_samples = resample_to_16k_fp32(wav.samples, wav.sample_rate); - break; - } - } - } - std::vector user_indices; - for (size_t i = 0; i < prompt.messages.size(); i++) { - if (prompt.messages[i].role == "user") user_indices.push_back(i); - } - auto& counts = handle->user_audio_counts; - for (size_t u = 0; u + 1 < user_indices.size(); u++) { - if (u < counts.size() && counts[u] > 0) { - prompt.messages[user_indices[u]].audio_soft_token_count = counts[u]; - } - } - if (!audio_samples.empty() && !user_indices.empty()) { - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, handle->model->get_config()); - prompt.audio_features = std::move(audio_prep.features); - prompt.audio_num_frames = audio_prep.num_frames; - size_t u = user_indices.size() - 1; - prompt.messages[user_indices[u]].audio_soft_token_count = audio_prep.num_soft_tokens; - if (counts.size() <= u) counts.resize(u + 1, 0); - counts[u] = audio_prep.num_soft_tokens; - } - } - - std::string formatted_tools; - if (Config::is_gemma_family(prompt.model_type)) { - formatted_tools = gemma::format_tools(prompt.tools, prompt.model_type == Config::ModelType::GEMMA4); - } else if (prompt.model_type == Config::ModelType::NEEDLE) { - formatted_tools = serialize_needle_tools(prompt.tools, tools_json, prompt.needle_name_map); - } else if (prompt.model_type == Config::ModelType::QWEN || prompt.model_type == Config::ModelType::QWEN3P5) { - formatted_tools = serialize_tools_for_template(prompt.tools); - } else { - formatted_tools = serialize_tools_json(prompt.tools); - } - - if (prompt.model_type == Config::ModelType::NEEDLE) { - if (apply_tool_constraints) { - setup_tool_constraints(handle, prompt.tools, prompt.options.force_tools, - prompt.options.temperature, prompt.needle_name_map); - } - - std::string query_text = cactus::engine::format_needle_query_text(prompt.messages); - prompt.tokens = tokenizer->encode(query_text); - auto suffix_tokens = encode_needle_tools_suffix(tokenizer, formatted_tools); - prompt.tokens.insert(prompt.tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); - } else { - std::string full_prompt = tokenizer->format_chat_prompt( - prompt.messages, - add_generation_prompt, - formatted_tools, - prompt.options.enable_thinking_if_supported - ); - if (full_prompt.find("ERROR:") == 0) { - throw std::runtime_error(full_prompt.substr(6)); - } - prompt.tokens = tokenizer->encode(full_prompt); - } - prompt.context_token_count = prompt.tokens.size(); - prompt.images = images_from_message(prompt.messages); - return prompt; -} - -PrefillResult do_prefill( - CactusModelHandle* handle, - const PreparedPrompt& prompt, - const std::vector& target_tokens -) { - PrefillResult result = {}; - bool has_images = prompt.has_images(); - - result.was_prefix = prompt_context_matches(handle, prompt); - result.was_exact_match = result.was_prefix && - target_tokens.size() == handle->processed_tokens.size(); - - if (result.was_exact_match) { - return result; - } - - std::vector tokens_to_process; - if (!result.was_prefix) { - reset_cache(handle); - tokens_to_process = target_tokens; - } else { - tokens_to_process.assign( - target_tokens.begin() + handle->processed_tokens.size(), - target_tokens.end() - ); - } - - if (tokens_to_process.size() > 1) { - std::vector prefill_tokens(tokens_to_process.begin(), tokens_to_process.end() - 1); - result.prefilled_count = prefill_tokens.size(); - if (has_images) { - std::vector delta_image_paths; - if (result.was_prefix) { - size_t cached_image_count = 0; - for (const auto& msg_imgs : handle->processed_images) { - cached_image_count += msg_imgs.size(); - } - delta_image_paths.assign( - prompt.image_paths.begin() + cached_image_count, - prompt.image_paths.end() - ); - } else { - delta_image_paths = prompt.image_paths; - } - handle->model->prefill_with_images(prefill_tokens, delta_image_paths); - } else { - handle->model->prefill(prefill_tokens, handle->model->get_prefill_chunk_size()); - } - result.remaining_tokens = {tokens_to_process.back()}; - } else { - result.remaining_tokens = tokens_to_process; - } - - return result; -} - -uint32_t decode( - std::unique_ptr& model, - const std::vector& tokens, - const InferenceOptions& options, - float* out_entropy -) { - return model->decode(tokens, options.temperature, options.top_p, options.top_k, - "", out_entropy, options.min_p, options.repetition_penalty); -} - -uint32_t generate_first_token( - CactusModelHandle* handle, - const PrefillResult& prefill_result, - const PreparedPrompt& prompt, - float* first_token_entropy -) { - if (prefill_result.was_exact_match || prefill_result.remaining_tokens.empty()) { - if (handle->processed_tokens.empty()) { - throw std::runtime_error("Cannot generate from empty prompt"); - } - return decode(handle->model, {handle->processed_tokens.back()}, prompt.options, first_token_entropy); - } - return decode(handle->model, prefill_result.remaining_tokens, prompt.options, first_token_entropy); -} - -std::string construct_prefill_response_json( - bool success, - const std::string* error, - size_t prefill_tokens, - double prefill_tps, - double total_time_ms -) { - std::ostringstream json; - json << "{"; - json << "\"success\":" << (success ? "true" : "false") << ","; - if (error) { - json << "\"error\":\"" << escape_json_string(*error) << "\","; - } else { - json << "\"error\":null,"; - } - json << "\"prefill_tokens\":" << prefill_tokens << ","; - json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - return json.str(); -} - -} // anonymous namespace - -extern "C" { - -int cactus_complete( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const char* tools_json, - cactus_token_callback callback, - void* user_data, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() ? - "Model not initialized. Check model path and files." : last_error_message; - CACTUS_LOG_ERROR("complete", error_msg); - handle_error_response(error_msg, response_buffer, buffer_size); - return -1; - } - - if (!messages_json || !response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("complete", "Invalid parameters: messages_json, response_buffer, or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - - auto* handle = static_cast(model); - handle->should_stop = false; - auto* tokenizer = handle->model->get_tokenizer(); - auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, true, true, pcm_buffer, pcm_buffer_size); - - CACTUS_LOG_DEBUG("complete", "Prompt tokens: " << prompt.tokens.size() - << ", max_tokens: " << prompt.options.max_tokens); - - bool has_images = prompt.has_images(); - bool has_audio = prompt.has_audio(); - const bool has_gemma4_mixed_media = prompt.model_type == Config::ModelType::GEMMA4 && has_images && has_audio; - auto decode_gemma4_mixed_media = [&](const std::vector& tokens, float* out_entropy) -> uint32_t { - auto* gemma4_mm = dynamic_cast(handle->model.get()); - if (!gemma4_mm) { - throw std::runtime_error("Gemma4 mixed-media decode requested on non-Gemma4 multimodal model"); - } - return gemma4_mm->decode_with_media( - tokens, - prompt.image_paths, - prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", out_entropy, - prompt.options.min_p, prompt.options.repetition_penalty - ); - }; - - auto stop_token_sequences = build_stop_sequences(tokenizer, prompt.options.stop_sequences, prompt.model_type, !prompt.tools.empty()); - - std::vector generated_tokens; - double time_to_first_token = 0.0; - float first_token_entropy = 0.0f; - uint32_t next_token; - size_t prompt_tokens; - - if ((has_gemma4_mixed_media || has_audio) && !handle->processed_tokens.empty()) { - auto& cache = handle->processed_tokens; - size_t common = 0; - size_t limit = std::min(cache.size(), prompt.tokens.size()); - while (common < limit && cache[common] == prompt.tokens[common]) common++; - if (common < cache.size()) { - CACTUS_LOG_WARN("complete", "KV cache diverges from new prompt at position " << common - << "/" << cache.size() << "; trimming and re-prefilling the divergent suffix"); - size_t kv_len = handle->model->get_cache_size(); - if (kv_len > common) { - handle->model->remove_thinking_tokens({{common, kv_len - common}}); - } - cache.resize(common); - } - } - - if (has_gemma4_mixed_media) { - prompt_tokens = prompt.tokens.size(); - next_token = decode_gemma4_mixed_media(prompt.tokens, &first_token_entropy); - } else if (has_audio) { - prompt_tokens = prompt.tokens.size(); - next_token = handle->model->decode_with_audio( - prompt.tokens, prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", &first_token_entropy, - prompt.options.min_p, prompt.options.repetition_penalty); - } else { - auto prefill_result = do_prefill(handle, prompt, prompt.tokens); - prompt_tokens = prefill_result.prefilled_count + prefill_result.remaining_tokens.size(); - next_token = generate_first_token(handle, prefill_result, prompt, &first_token_entropy); - } - - handle->processed_tokens = prompt.tokens; - handle->processed_images = prompt.images; - - auto token_end = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(token_end - start_time).count() / 1000.0; - - float confidence = 1.0f - first_token_entropy; - bool cloud_used = false; - std::string cloud_error; - std::future cloud_future; - bool cloud_future_started = false; - const bool cloud_eligible = prompt.options.auto_handoff && (!has_images || prompt.options.handoff_with_images); - - auto maybe_start_cloud_handoff = [&](const std::string& local_output_hint, - const std::vector& local_calls_hint) { - if (!cloud_eligible || cloud_future_started) { - return; - } - CloudCompletionRequest request; - request.messages = prompt.messages; - request.tools = prompt.tools; - request.local_output = local_output_hint; - request.local_function_calls = local_calls_hint; - request.has_images = has_images; - request.has_audio = has_audio; - if (has_audio && pcm_buffer != nullptr && pcm_buffer_size > 0) { - request.audio_pcm.assign(pcm_buffer, pcm_buffer + pcm_buffer_size); - } - request.cloud_key = resolve_cloud_api_key(nullptr); - - cloud_future_started = true; - cloud_future = std::async(std::launch::async, [request, &prompt]() { - return cloud_complete_request(request, static_cast(prompt.options.cloud_timeout_ms)); - }); - }; - - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff("", {}); - } - - generated_tokens.push_back(next_token); - handle->processed_tokens.push_back(next_token); - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->update_tool_constraints(next_token); - } - - EntropyState entropy; - { - size_t cfg_window = handle->model->get_config().default_rolling_entropy_window; - if (cfg_window > 0) entropy.window_size = cfg_window; - } - entropy.add(first_token_entropy); - - if (!matches_stop_sequence(generated_tokens, stop_token_sequences)) { - if (callback) { - std::string new_text = tokenizer->decode({next_token}); - callback(new_text.c_str(), next_token, user_data); - } - - for (size_t i = 1; i < prompt.options.max_tokens; i++) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - if (has_gemma4_mixed_media) { - next_token = decode_gemma4_mixed_media(handle->processed_tokens, &token_entropy); - } else if (has_audio) { - next_token = handle->model->decode_with_audio( - handle->processed_tokens, prompt.audio_features, - prompt.options.temperature, prompt.options.top_p, prompt.options.top_k, - "", &token_entropy, - prompt.options.min_p, prompt.options.repetition_penalty); - } else { - next_token = decode(handle->model, {next_token}, prompt.options, &token_entropy); - } - handle->processed_tokens.push_back(next_token); - generated_tokens.push_back(next_token); - - entropy.add(token_entropy); - - if (entropy.rolling_confidence() < prompt.options.confidence_threshold) { - entropy.spike_handoff = true; - maybe_start_cloud_handoff("", {}); - } - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->update_tool_constraints(next_token); - } - - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { - trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); - break; - } - - if (callback) { - std::string new_text = tokenizer->decode({next_token}); - callback(new_text.c_str(), next_token, user_data); - } - } - } else { - trim_stop_suffix(generated_tokens, stop_token_sequences, prompt.options.include_stop_sequences); - } - - confidence = entropy.mean_confidence(); - - if (prompt.options.force_tools && !prompt.tools.empty()) { - handle->model->clear_tool_constraints(); - } - - if (prompt.model_type == Config::ModelType::GEMMA4 && prompt.options.enable_thinking_if_supported && !generated_tokens.empty()) { - strip_thinking_from_cache(handle, generated_tokens, prompt.tokens.size()); - } - - if (prompt.model_type == Config::ModelType::GEMMA4) { - handle->model->compact_kv_cache(); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - size_t completion_tokens = generated_tokens.size(); - double decode_time_ms = total_time_ms - time_to_first_token; - double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - std::string response_text = tokenizer->decode(generated_tokens); - - std::string regular_response; - std::vector function_calls; - parse_function_calls_from_response(response_text, regular_response, function_calls); - - if (prompt.model_type == Config::ModelType::NEEDLE) { - needle::restore_tool_names(function_calls, prompt.needle_name_map); - } - - std::string thinking_text; - if (prompt.model_type == Config::ModelType::GEMMA4 || prompt.options.enable_thinking_if_supported) { - std::string stripped_content; - strip_thinking_block(regular_response, thinking_text, stripped_content); - regular_response = stripped_content; - if (!prompt.options.enable_thinking_if_supported) { - thinking_text.clear(); - } - } - - if (confidence < prompt.options.confidence_threshold) { - maybe_start_cloud_handoff(regular_response, function_calls); - } - - std::string local_completion = regular_response; - if (local_completion.empty() && function_calls.empty()) { - local_completion = response_text; - } - std::string primary_response = local_completion; - std::vector primary_function_calls = function_calls; - - if (cloud_future_started) { - auto status = cloud_future.wait_for(std::chrono::milliseconds(prompt.options.cloud_timeout_ms)); - if (status == std::future_status::ready) { - CloudCompletionResult cloud_result = cloud_future.get(); - if (cloud_result.ok && (!cloud_result.response.empty() || !cloud_result.function_calls.empty())) { - cloud_used = true; - if (!cloud_result.response.empty()) { - primary_response = cloud_result.response; - } - if (!cloud_result.function_calls.empty()) { - primary_function_calls = cloud_result.function_calls; - } - } else { - cloud_error = cloud_result.error.empty() ? "cloud completion failed" : cloud_result.error; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion failed, falling back to local output: " << cloud_error); - } - } else { - cloud_error = "timeout"; - CACTUS_LOG_WARN("cloud_handoff", "Cloud completion timed out, falling back to local output: " << cloud_error); - } - } - - const bool handoff_succeeded = cloud_used; - std::string result = construct_response_json(primary_response, primary_function_calls, time_to_first_token, - total_time_ms, prefill_tps, decode_tps, prompt_tokens, - completion_tokens, confidence, handoff_succeeded, - thinking_text); - - if (result.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - - std::string function_calls_json = serialize_function_calls(primary_function_calls); - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = true; - metrics.cloud_handoff = handoff_succeeded; - metrics.ttft_ms = time_to_first_token; - metrics.prefill_tps = prefill_tps; - metrics.decode_tps = decode_tps; - metrics.response_time_ms = total_time_ms; - metrics.confidence = confidence; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = prompt_tokens; - metrics.decode_tokens = completion_tokens; - metrics.error_message = nullptr; - metrics.function_calls_json = nullptr; - cactus::telemetry::recordCompletion(handle->model_name.c_str(), metrics); - - return static_cast(result.length()); - - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("complete", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = false; - metrics.cloud_handoff = false; - metrics.ttft_ms = 0.0; - metrics.prefill_tps = 0.0; - metrics.decode_tps = 0.0; - metrics.response_time_ms = 0.0; - metrics.confidence = 0.0; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = 0; - metrics.decode_tokens = 0; - metrics.error_message = e.what(); - metrics.function_calls_json = nullptr; - auto* h = static_cast(model); - cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); - - return -1; - } catch (...) { - CACTUS_LOG_ERROR("complete", "Unknown exception during completion"); - handle_error_response("Unknown error during completion", response_buffer, buffer_size); - - cactus::telemetry::CompletionMetrics metrics{}; - metrics.success = false; - metrics.cloud_handoff = false; - metrics.ttft_ms = 0.0; - metrics.prefill_tps = 0.0; - metrics.decode_tps = 0.0; - metrics.response_time_ms = 0.0; - metrics.confidence = 0.0; - metrics.ram_usage_mb = get_ram_usage_mb(); - metrics.prefill_tokens = 0; - metrics.decode_tokens = 0; - metrics.error_message = "Unknown error during completion"; - metrics.function_calls_json = nullptr; - auto* h = static_cast(model); - cactus::telemetry::recordCompletion(h ? h->model_name.c_str() : "unknown", metrics); - - return -1; - } -} - -int cactus_prefill( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const char* tools_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() - ? "Model not initialized. Check model path and files." - : last_error_message; - if (response_buffer && buffer_size > 0) { - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - } - return -1; - } - - if (!messages_json || !response_buffer || buffer_size == 0) { - std::string error_msg = "Invalid parameters"; - if (response_buffer && buffer_size > 0) { - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - } - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - - auto* handle = static_cast(model); - auto prompt = prepare_prompt(handle, messages_json, options_json, tools_json, false, false, pcm_buffer, pcm_buffer_size); - - std::vector context_tokens(prompt.tokens.begin(), prompt.tokens.begin() + prompt.context_token_count); - auto prefill_result = do_prefill(handle, prompt, context_tokens); - - if (!prefill_result.was_exact_match) { - handle->processed_tokens = context_tokens; - if (!handle->processed_tokens.empty()) { - handle->processed_tokens.pop_back(); - } - } - handle->processed_images = prompt.images; - - auto end_time = std::chrono::high_resolution_clock::now(); - double elapsed_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - double prefill_tps = (prefill_result.prefilled_count > 0 && elapsed_ms > 0.0) - ? (static_cast(prefill_result.prefilled_count) * 1000.0) / elapsed_ms - : 0.0; - - std::string result = construct_prefill_response_json(true, nullptr, prefill_result.prefilled_count, prefill_tps, elapsed_ms); - if (result.size() >= buffer_size) { - std::string error_msg = "Response buffer too small"; - std::string error_json = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (error_json.size() < buffer_size) { - std::strcpy(response_buffer, error_json.c_str()); - } - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return static_cast(result.size()); - } catch (const std::exception& e) { - std::string error_msg = e.what(); - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - return -1; - } catch (...) { - std::string error_msg = "Unknown error during prefill"; - std::string result = construct_prefill_response_json(false, &error_msg, 0, 0.0, 0.0); - if (result.size() < buffer_size) { - std::strcpy(response_buffer, result.c_str()); - } - return -1; - } -} - -int cactus_tokenize( - cactus_model_t model, - const char* text, - uint32_t* token_buffer, - size_t token_buffer_len, - size_t* out_token_len -) { - if (!model || !text || !out_token_len) return -1; - - try { - auto* handle = static_cast(model); - auto* tokenizer = handle->model->get_tokenizer(); - - std::vector toks = tokenizer->encode(std::string(text)); - *out_token_len = toks.size(); - - if (!token_buffer || token_buffer_len == 0) return 0; - if (token_buffer_len < toks.size()) return -2; - - std::memcpy(token_buffer, toks.data(), toks.size() * sizeof(uint32_t)); - return 0; - } catch (...) { - return -1; - } -} - -int cactus_score_window( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - size_t start, - size_t end, - size_t context, - char* response_buffer, - size_t buffer_size -) { - if (!model || !tokens || token_len == 0 || !response_buffer || buffer_size == 0) { - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - - try { - auto* handle = static_cast(model); - - std::vector vec(tokens, tokens + token_len); - - size_t scored = 0; - double logprob = handle->model->score_tokens_window_logprob(vec, start, end, context, &scored); - - std::ostringstream oss; - oss << "{" - << "\"success\":true," - << "\"logprob\":" << std::setprecision(10) << logprob << "," - << "\"tokens\":" << scored - << "}"; - - std::string result = oss.str(); - if (result.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return (int)result.size(); - - } catch (const std::exception& e) { - handle_error_response(e.what(), response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_diarize.cpp b/cactus/ffi/cactus_diarize.cpp deleted file mode 100644 index 358c7aa37..000000000 --- a/cactus/ffi/cactus_diarize.cpp +++ /dev/null @@ -1,200 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr int DIARIZE_MAX_SPEAKERS = 3; - -struct DiarizeOptions { - size_t step_samples = 16000; - float threshold = -1.0f; - int num_speakers = -1; - int min_speakers = -1; - int max_speakers = -1; - bool raw_powerset = false; -}; - -static DiarizeOptions parse_diarize_options(const std::string& json) { - DiarizeOptions opts; - if (json.empty()) return opts; - - size_t pos; - - pos = json.find("\"step_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - int v = std::stoi(json.substr(pos)); - if (v > 0) opts.step_samples = static_cast(v) * 16; - } - - pos = json.find("\"threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"num_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.num_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"min_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.min_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"max_speakers\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.max_speakers = std::stoi(json.substr(pos)); - } - - pos = json.find("\"raw_powerset\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - opts.raw_powerset = (json.substr(pos, 4) == "true"); - } - - return opts; -} - -static void apply_speaker_filter(std::vector& scores, size_t total_frames, const DiarizeOptions& opts) { - int keep = DIARIZE_MAX_SPEAKERS; - if (opts.num_speakers > 0) keep = std::min(opts.num_speakers, DIARIZE_MAX_SPEAKERS); - if (opts.max_speakers > 0) keep = std::min(keep, opts.max_speakers); - if (opts.min_speakers > 0) keep = std::max(keep, std::min(opts.min_speakers, DIARIZE_MAX_SPEAKERS)); - if (keep >= DIARIZE_MAX_SPEAKERS) return; - - float activity[DIARIZE_MAX_SPEAKERS] = {}; - for (size_t f = 0; f < total_frames; ++f) - for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) - activity[s] += scores[f * DIARIZE_MAX_SPEAKERS + s]; - - int order[DIARIZE_MAX_SPEAKERS] = {0, 1, 2}; - std::sort(order, order + DIARIZE_MAX_SPEAKERS, [&](int a, int b) { return activity[a] > activity[b]; }); - - bool active[DIARIZE_MAX_SPEAKERS] = {}; - for (int i = 0; i < keep; ++i) active[order[i]] = true; - - for (size_t f = 0; f < total_frames; ++f) - for (int s = 0; s < DIARIZE_MAX_SPEAKERS; ++s) - if (!active[s]) scores[f * DIARIZE_MAX_SPEAKERS + s] = 0.0f; -} - -static void apply_threshold(std::vector& scores, float threshold) { - for (float& v : scores) - v = (v >= threshold) ? v : 0.0f; -} - -extern "C" { - -int cactus_diarize( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("diarize", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* pyannote = dynamic_cast(handle->model.get()); - if (!pyannote) { - CACTUS_LOG_ERROR("diarize", "Model is not a PyAnnote diarization model"); - handle_error_response("Model is not a PyAnnote diarization model", response_buffer, buffer_size); - return -1; - } - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - const DiarizeOptions opts = parse_diarize_options(options_json ? options_json : ""); - auto scores = pyannote->diarize(audio.data(), audio.size(), opts.step_samples, opts.raw_powerset); - audio = {}; - - const int num_channels = opts.raw_powerset ? 7 : DIARIZE_MAX_SPEAKERS; - const size_t total_frames = scores.size() / num_channels; - - if (!opts.raw_powerset) { - if (opts.num_speakers > 0 || opts.min_speakers > 0 || opts.max_speakers > 0) - apply_speaker_filter(scores, total_frames, opts); - - if (opts.threshold >= 0.0f) - apply_threshold(scores, opts.threshold); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"num_speakers\":" << num_channels << ","; - json << "\"scores\":["; - json << std::fixed << std::setprecision(6); - for (size_t i = 0; i < scores.size(); ++i) { - if (i > 0) json << ","; - json << scores[i]; - } - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during diarization: " + std::string(e.what()); - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during diarization"; - CACTUS_LOG_ERROR("diarize", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_embed.cpp b/cactus/ffi/cactus_embed.cpp deleted file mode 100644 index 2ce85f0ca..000000000 --- a/cactus/ffi/cactus_embed.cpp +++ /dev/null @@ -1,192 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../../libs/audio/wav.h" -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::get_whisper_spectrogram_config; -using cactus::audio::init_whisper_mel_filters; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::normalize_whisper_mel; -using cactus::audio::trim_mel_frames; - -static std::vector compute_mel_from_wav(const std::string& wav_path, bool is_parakeet, size_t mel_bins) { - AudioFP32 audio = load_wav(wav_path); - std::vector waveform_16k = resample_to_16k_fp32(audio.samples, audio.sample_rate); - - auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); - const size_t num_mel_filters = std::max(1, mel_bins); - - AudioProcessor ap; - if (is_parakeet) { - ap.init_mel_filters(cfg.n_fft / 2 + 1, num_mel_filters, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); - } else { - init_whisper_mel_filters(ap, cfg, num_mel_filters); - } - const size_t waveform_samples = waveform_16k.size(); - if (is_parakeet) { - apply_preemphasis(waveform_16k, 0.97f); - } - std::vector mel = ap.compute_spectrogram(waveform_16k, cfg); - - if (mel.empty()) return mel; - if (is_parakeet) { - normalize_parakeet_log_mel(mel, num_mel_filters); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(mel, num_mel_filters, valid_frames); - return mel; - } - - const bool is_v3 = num_mel_filters > 80; - return normalize_whisper_mel(mel, num_mel_filters, is_v3); -} - -extern "C" { - -int cactus_embed( - cactus_model_t model, - const char* text, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim, - bool normalize -) { - if (!model || !text || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("embed", "Invalid parameters for text embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - auto* tokenizer = handle->model->get_tokenizer(); - - std::vector tokens = tokenizer->encode(text); - if (tokens.empty()) { - CACTUS_LOG_ERROR("embed", "Tokenization produced empty result"); - return -1; - } - - std::vector embeddings = handle->model->get_embeddings(tokens, true, normalize); - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes, got " << buffer_size); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during embedding"; - CACTUS_LOG_ERROR("embed", last_error_message); - return -1; - } -} - -int cactus_image_embed( - cactus_model_t model, - const char* image_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -) { - if (!model || !image_path || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("image_embed", "Invalid parameters for image embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - - CACTUS_LOG_DEBUG("image_embed", "Processing image: " << image_path); - std::vector embeddings = handle->model->get_image_embeddings(image_path); - if (embeddings.empty()) { - CACTUS_LOG_ERROR("image_embed", "Image embedding returned empty result"); - return -1; - } - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("image_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("image_embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during image embedding"; - CACTUS_LOG_ERROR("image_embed", last_error_message); - return -1; - } -} - -int cactus_audio_embed( - cactus_model_t model, - const char* audio_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -) { - if (!model || !audio_path || !embeddings_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("audio_embed", "Invalid parameters for audio embedding"); - return -1; - } - - try { - auto* handle = static_cast(model); - - CACTUS_LOG_DEBUG("audio_embed", "Processing audio: " << audio_path); - const bool is_parakeet = - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - auto mel_bins = compute_mel_from_wav(audio_path, is_parakeet, handle->model->get_config().num_mel_bins); - if (mel_bins.empty()) { - last_error_message = "Failed to compute mel spectrogram"; - CACTUS_LOG_ERROR("audio_embed", last_error_message << " for: " << audio_path); - return -1; - } - - std::vector embeddings = handle->model->get_audio_embeddings(mel_bins); - if (embeddings.empty()) { - CACTUS_LOG_ERROR("audio_embed", "Audio embedding returned empty result"); - return -1; - } - if (embeddings.size() * sizeof(float) > buffer_size) { - CACTUS_LOG_ERROR("audio_embed", "Buffer too small: need " << embeddings.size() * sizeof(float) << " bytes"); - return -2; - } - - std::memcpy(embeddings_buffer, embeddings.data(), embeddings.size() * sizeof(float)); - if (embedding_dim) *embedding_dim = embeddings.size(); - - return static_cast(embeddings.size()); - - } catch (const std::exception& e) { - last_error_message = e.what(); - CACTUS_LOG_ERROR("audio_embed", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during audio embedding"; - CACTUS_LOG_ERROR("audio_embed", last_error_message); - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_embed_speaker.cpp b/cactus/ffi/cactus_embed_speaker.cpp deleted file mode 100644 index ddb42707e..000000000 --- a/cactus/ffi/cactus_embed_speaker.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::get_wespeaker_spectrogram_config; - -extern "C" { - -int cactus_embed_speaker( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const float* mask_weights, - size_t mask_num_frames -) { - (void)options_json; - if (validate_audio_params("embed_speaker", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* wespeaker = dynamic_cast(handle->model.get()); - if (!wespeaker) { - CACTUS_LOG_ERROR("embed_speaker", "Model is not a WeSpeaker embedding model"); - handle_error_response("Model is not a WeSpeaker embedding model", response_buffer, buffer_size); - return -1; - } - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - static const auto s_cfg = get_wespeaker_spectrogram_config(); - static AudioProcessor s_ap = []() { - AudioProcessor p; - p.init_mel_filters(get_wespeaker_spectrogram_config().n_fft / 2 + 1, - 80, 20.0f, 8000.0f, 16000, nullptr, "htk"); - return p; - }(); - - std::vector mel = s_ap.compute_spectrogram(audio, s_cfg); - audio = {}; - - static constexpr size_t NUM_MEL_BINS = 80; - const size_t num_frames = mel.size() / NUM_MEL_BINS; - - for (size_t m = 0; m < NUM_MEL_BINS; ++m) { - float* row = mel.data() + m * num_frames; - float sum = 0.0f; - for (size_t t = 0; t < num_frames; ++t) sum += row[t]; - const float mean = sum / static_cast(num_frames); - for (size_t t = 0; t < num_frames; ++t) row[t] -= mean; - } - - std::vector embedding; - if (mask_weights && mask_num_frames > 0) { - embedding = wespeaker->embed(mel.data(), mel.size(), mask_weights, mask_num_frames); - } else { - embedding = wespeaker->embed(mel.data(), mel.size()); - } - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"embedding\":["; - json << std::fixed << std::setprecision(6); - for (size_t i = 0; i < embedding.size(); ++i) { - if (i > 0) json << ","; - json << embedding[i]; - } - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during speaker embedding: " + std::string(e.what()); - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during speaker embedding"; - CACTUS_LOG_ERROR("embed_speaker", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_ffi.h b/cactus/ffi/cactus_ffi.h deleted file mode 100644 index 163e1805c..000000000 --- a/cactus/ffi/cactus_ffi.h +++ /dev/null @@ -1,454 +0,0 @@ -#ifndef CACTUS_FFI_H -#define CACTUS_FFI_H - -#include -#include -#include - -#if __GNUC__ >= 4 - #define CACTUS_FFI_EXPORT __attribute__((visibility("default"))) - #define CACTUS_FFI_LOCAL __attribute__((visibility("hidden"))) -#else - #define CACTUS_FFI_EXPORT - #define CACTUS_FFI_LOCAL -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* cactus_model_t; -typedef void* cactus_index_t; -typedef void* cactus_stream_transcribe_t; - -typedef void (*cactus_token_callback)(const char* token, uint32_t token_id, void* user_data); - -CACTUS_FFI_EXPORT cactus_model_t cactus_init( - const char* model_path, - const char* corpus_dir, // optional: NULL if no RAG corpus - bool cache_index // false = always rebuild index, true = load cached if available -); - -CACTUS_FFI_EXPORT void cactus_destroy(cactus_model_t model); -CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model); -CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model); - -CACTUS_FFI_EXPORT int cactus_complete( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const char* tools_json, // optional - cactus_token_callback callback, // optional - void* user_data, // optional - const uint8_t* pcm_buffer, // optional: NULL when not used - size_t pcm_buffer_size // optional: 0 when not used -); - -CACTUS_FFI_EXPORT int cactus_prefill( - cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const char* tools_json, // optional - const uint8_t* pcm_buffer, // optional: NULL when not used - size_t pcm_buffer_size // optional: 0 when not used -); - -CACTUS_FFI_EXPORT int cactus_tokenize( - cactus_model_t model, - const char* text, - uint32_t* token_buffer, - size_t token_buffer_len, - size_t* out_token_len -); - -CACTUS_FFI_EXPORT int cactus_score_window( - cactus_model_t model, - const uint32_t* tokens, - size_t token_len, - size_t start, - size_t end, - size_t context, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_transcribe( - cactus_model_t model, - const char* audio_file_path, // NULL if using pcm_buffer - const char* prompt, - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - cactus_token_callback callback, // optional - void* user_data, // optional - const uint8_t* pcm_buffer, // NULL if using audio_file_path - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_detect_language( - cactus_model_t model, - const char* audio_file_path, // NULL if using pcm_buffer - char* response_buffer, - size_t buffer_size, - const char* options_json, // optional - const uint8_t* pcm_buffer, // NULL if using audio_file_path - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT cactus_stream_transcribe_t cactus_stream_transcribe_start( - cactus_model_t model, - const char* options_json // optional -); - -CACTUS_FFI_EXPORT int cactus_stream_transcribe_process( - cactus_stream_transcribe_t stream, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_stream_transcribe_stop( - cactus_stream_transcribe_t stream, - char* response_buffer, - size_t buffer_size -); - -CACTUS_FFI_EXPORT int cactus_embed( - cactus_model_t model, - const char* text, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim, - bool normalize -); - -CACTUS_FFI_EXPORT int cactus_image_embed( - cactus_model_t model, - const char* image_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_audio_embed( - cactus_model_t model, - const char* audio_path, - float* embeddings_buffer, - size_t buffer_size, - size_t* embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_vad( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_diarize( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -); - -CACTUS_FFI_EXPORT int cactus_embed_speaker( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const float* mask_weights, - size_t mask_num_frames -); - -CACTUS_FFI_EXPORT int cactus_rag_query( - cactus_model_t model, - const char* query, - char* response_buffer, - size_t buffer_size, - size_t top_k -); - -CACTUS_FFI_EXPORT cactus_index_t cactus_index_init( - const char* index_dir, - size_t embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_index_add( - cactus_index_t index, - const int* ids, - const char** documents, - const char** metadatas, // optional: can be NULL - const float** embeddings, - size_t count, - size_t embedding_dim -); - -CACTUS_FFI_EXPORT int cactus_index_delete( - cactus_index_t index, - const int* ids, - size_t ids_count -); - -CACTUS_FFI_EXPORT int cactus_index_get( - cactus_index_t index, - const int* ids, - size_t ids_count, - char** document_buffers, - size_t* document_buffer_sizes, - char** metadata_buffers, - size_t* metadata_buffer_sizes, - float** embedding_buffers, - size_t* embedding_buffer_sizes -); - -CACTUS_FFI_EXPORT int cactus_index_query( - cactus_index_t index, - const float** embeddings, - size_t embeddings_count, - size_t embedding_dim, - const char* options_json, // optional - int** id_buffers, - size_t* id_buffer_sizes, - float** score_buffers, - size_t* score_buffer_sizes -); - -CACTUS_FFI_EXPORT int cactus_index_compact(cactus_index_t index); -CACTUS_FFI_EXPORT void cactus_index_destroy(cactus_index_t index); - -CACTUS_FFI_EXPORT const char* cactus_get_last_error(void); - -// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE -CACTUS_FFI_EXPORT void cactus_log_set_level(int level); - -typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); -CACTUS_FFI_EXPORT void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); - -CACTUS_FFI_EXPORT void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); -CACTUS_FFI_EXPORT void cactus_set_app_id(const char* app_id); -CACTUS_FFI_EXPORT void cactus_telemetry_flush(void); -CACTUS_FFI_EXPORT void cactus_telemetry_shutdown(void); - -// cactus graph export -typedef void* cactus_graph_t; -typedef uint64_t cactus_node_t; - -typedef struct { - int32_t precision; - size_t rank; - size_t shape[8]; - size_t num_elements; - size_t byte_size; -} cactus_tensor_info_t; - -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_create(void); -CACTUS_FFI_EXPORT void cactus_graph_destroy(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_hard_reset(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_save(cactus_graph_t graph, const char* filename); -CACTUS_FFI_EXPORT cactus_graph_t cactus_graph_load(const char* filename); - -CACTUS_FFI_EXPORT int cactus_graph_input( - cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, -cactus_node_t* out_node); - -CACTUS_FFI_EXPORT int cactus_graph_set_input( - cactus_graph_t graph, cactus_node_t node, const void* data, int32_t -precision); -CACTUS_FFI_EXPORT int cactus_graph_set_external_input( - cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision); - -CACTUS_FFI_EXPORT int cactus_graph_precision_cast( - cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_quantize_activations( - cactus_graph_t graph, cactus_node_t input, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, -cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_divide(cactus_graph_t graph, cactus_node_t -a, cactus_node_t b, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, -float exponent, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_view( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_flatten( - cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, -cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_reshape( - cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose( - cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_transpose_n( - cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_slice( - cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_index( - cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_concat( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, -cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_cat( - cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t -axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_matmul( - cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gather( - cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_tensor( - cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_embedding_from_file( - cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_embeddings( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_mmap_weights( - cactus_graph_t graph, const char* filename, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_bilinear_interpolation( - cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_set_grouped_scales( - cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr); -CACTUS_FFI_EXPORT int cactus_graph_set_interleaved( - cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n); -CACTUS_FFI_EXPORT int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node); -CACTUS_FFI_EXPORT int cactus_graph_release_all_weight_pages(cactus_graph_t graph); - -CACTUS_FFI_EXPORT int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_layernorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_groupnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_batchnorm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rms_norm( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rope_gptj( - cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_rel_pos_bias( - cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_attention_int8_hybrid( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_conv1d_causal( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_k7s3( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_same_depthwise_k9( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv1d_pointwise( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_depthwise_k3s2p1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_conv2d_pointwise_1x1( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_lstm_cell( - cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_decode( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gated_deltanet_prefill( - cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_stft( - cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_altup_predict( - cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_altup_correct( - cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_gaussian_topk( - cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_gated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_moe_layer_ungated( - cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out); - -CACTUS_FFI_EXPORT int cactus_graph_sample( - cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_scatter_topk( - cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_persistent( - cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out); -CACTUS_FFI_EXPORT int cactus_graph_is_populated( - cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated); -CACTUS_FFI_EXPORT int cactus_graph_invalidate_persistent( - cactus_graph_t graph, cactus_node_t persistent_node); - -CACTUS_FFI_EXPORT int cactus_graph_execute(cactus_graph_t graph); -CACTUS_FFI_EXPORT int cactus_graph_get_output_ptr(cactus_graph_t graph, -cactus_node_t node, void** out_ptr); -CACTUS_FFI_EXPORT int cactus_graph_get_output_info(cactus_graph_t graph, -cactus_node_t node, cactus_tensor_info_t* out_info); - -#ifdef __cplusplus -} -#endif - -#endif // CACTUS_FFI_H diff --git a/cactus/ffi/cactus_graph.cpp b/cactus/ffi/cactus_graph.cpp deleted file mode 100644 index f0aa77f37..000000000 --- a/cactus/ffi/cactus_graph.cpp +++ /dev/null @@ -1,1257 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../graph/graph.h" -#include -#include - -using namespace cactus::ffi; - -struct GraphHandle { - CactusGraph graph; -}; - -static GraphHandle* as_graph(cactus_graph_t g) { - return reinterpret_cast(g); -} - -static int fail_invalid(const char* msg) { - last_error_message = msg; - return -1; -} - -#define GRAPH_WRAP_UNARY(fn_name, method_name) \ -int fn_name(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { \ - if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ - try { \ - *out = static_cast(as_graph(graph)->graph.method_name(static_cast(x))); \ - return 0; \ - } catch (const std::exception& e) { \ - last_error_message = e.what(); \ - return -1; \ - } \ -} - -#define GRAPH_WRAP_BINARY(fn_name, method_name) \ -int fn_name(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { \ - if (!graph || !out) return fail_invalid("Invalid args to " #fn_name); \ - try { \ - *out = static_cast(as_graph(graph)->graph.method_name(static_cast(a), static_cast(b))); \ - return 0; \ - } catch (const std::exception& e) { \ - last_error_message = e.what(); \ - return -1; \ - } \ -} - -extern "C" { - -cactus_graph_t cactus_graph_create(void) { - try { - return reinterpret_cast(new GraphHandle()); - } catch (const std::exception& e) { - last_error_message = e.what(); - return nullptr; - } catch (...) { - last_error_message = "Unknown error creating graph"; - return nullptr; - } -} - -void cactus_graph_destroy(cactus_graph_t graph) { - delete as_graph(graph); -} - -int cactus_graph_hard_reset(cactus_graph_t graph) { - if (!graph) { - last_error_message = "Invalid args to cactus_graph_hard_reset"; - return -1; - } - try { - as_graph(graph)->graph.hard_reset(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_save(cactus_graph_t graph, const char* filename) { - if (!graph || !filename) { - last_error_message = "Invalid args to cactus_graph_save"; - return -1; - } - try { - as_graph(graph)->graph.save(filename); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - - -cactus_graph_t cactus_graph_load(const char* filename) { - if (!filename) { - last_error_message = "Invalid args to cactus_graph_load"; - return nullptr; - } - try { - auto handle = std::make_unique(); - handle->graph = CactusGraph::load(std::string(filename)); - return reinterpret_cast(handle.release()); - } catch (const std::exception& e) { - last_error_message = e.what(); - return nullptr; - } catch (...) { - last_error_message = "Unknown error loading graph"; - return nullptr; - } -} - -int cactus_graph_input(cactus_graph_t graph, const size_t* shape, size_t rank, int32_t precision, cactus_node_t* out_node) { - if (!graph || !shape || rank == 0 || !out_node) { - last_error_message = "Invalid args to cactus_graph_input"; - return -1; - } - try { - std::vector s(shape, shape + rank); - auto id = as_graph(graph)->graph.input(s, static_cast(precision)); - *out_node = static_cast(id); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_input(cactus_graph_t graph, cactus_node_t node, const void* data, int32_t precision) { - if (!graph || !data) { - last_error_message = "Invalid args to cactus_graph_set_input"; - return -1; - } - try { - as_graph(graph)->graph.set_input(static_cast(node), data, static_cast(precision)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_external_input(cactus_graph_t graph, cactus_node_t node, void* data, int32_t precision) { - if (!graph || !data) { - return fail_invalid("Invalid args to cactus_graph_set_external_input"); - } - try { - as_graph(graph)->graph.set_external_input(static_cast(node), data, static_cast(precision)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_precision_cast(cactus_graph_t graph, cactus_node_t input, int32_t target_precision, cactus_node_t* out) { - if (!graph || !out) { - return fail_invalid("Invalid args to cactus_graph_precision_cast"); - } - try { - *out = static_cast(as_graph(graph)->graph.precision_cast(static_cast(input), static_cast(target_precision))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_quantize_activations(cactus_graph_t graph, cactus_node_t input, cactus_node_t* out) { - if (!graph || !out) { - return fail_invalid("Invalid args to cactus_graph_quantize_activations"); - } - try { - *out = static_cast(as_graph(graph)->graph.quantize_activations(static_cast(input))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_add(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_add"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.add(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_add_clipped(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_add_clipped"); - try { - *out = static_cast(as_graph(graph)->graph.add_clipped(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_subtract(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_subtract"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.subtract(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_multiply(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_multiply"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.multiply(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_divide(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_divide"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.divide(static_cast(a), static_cast(b))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_add(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_add"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_add(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_subtract(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_subtract"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_subtract(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_multiply(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_multiply"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_multiply(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_divide(cactus_graph_t graph, cactus_node_t x, float value, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_divide"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_divide(static_cast(x), value)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_exp(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_exp"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_exp(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_sqrt(cactus_graph_t graph, cactus_node_t x, cactus_node_t *out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_scalar_sqrt"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.scalar_sqrt(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_cos(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_cos"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_cos(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_sin(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_sin"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_sin(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scalar_log(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scalar_log"); - try { - *out = static_cast(as_graph(graph)->graph.scalar_log(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_abs(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_abs"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.abs(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_pow(cactus_graph_t graph, cactus_node_t x, float exponent, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_pow"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.pow(static_cast(x), exponent)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_view(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { - if (!graph || !shape || rank == 0 || !out) { - last_error_message = "Invalid args to cactus_graph_view"; - return -1; - } - try { - std::vector s(shape, shape + rank); - *out = static_cast(as_graph(graph)->graph.view(static_cast(x), s)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_flatten(cactus_graph_t graph, cactus_node_t x, int32_t start_dim, int32_t end_dim, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_flatten"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.flatten(static_cast(x), start_dim, end_dim)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_reshape(cactus_graph_t graph, cactus_node_t x, const size_t* shape, size_t rank, cactus_node_t* out) { - if (!graph || !shape || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_reshape"); - try { - std::vector s(shape, shape + rank); - *out = static_cast(as_graph(graph)->graph.reshape(static_cast(x), s)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_transpose(cactus_graph_t graph, cactus_node_t x, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_transpose"); - try { - *out = static_cast(as_graph(graph)->graph.transpose(static_cast(x), static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_transpose_n(cactus_graph_t graph, cactus_node_t x, const size_t* permutation, size_t rank, int32_t backend, cactus_node_t* out) { - if (!graph || !permutation || rank == 0 || !out) return fail_invalid("Invalid args to cactus_graph_transpose_n"); - try { - std::vector p(permutation, permutation + rank); - *out = static_cast(as_graph(graph)->graph.transposeN(static_cast(x), p, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_slice(cactus_graph_t graph, cactus_node_t x, int32_t axis, size_t start, size_t length, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_slice"); - try { - *out = static_cast(as_graph(graph)->graph.slice(static_cast(x), axis, start, length)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_index(cactus_graph_t graph, cactus_node_t x, size_t index_value, int32_t dim, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_index"); - try { - *out = static_cast(as_graph(graph)->graph.index(static_cast(x), index_value, dim)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sum(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sum"); - try { - *out = static_cast(as_graph(graph)->graph.sum(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mean(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_mean"); - try { - *out = static_cast(as_graph(graph)->graph.mean(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_variance(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_variance"); - try { - *out = static_cast(as_graph(graph)->graph.variance(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_min(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_min"); - try { - *out = static_cast(as_graph(graph)->graph.min(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_max(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_max"); - try { - *out = static_cast(as_graph(graph)->graph.max(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_concat(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, int32_t axis, cactus_node_t* out) { - if (!graph || !out) { - last_error_message = "Invalid args to cactus_graph_concat"; - return -1; - } - try { - *out = static_cast(as_graph(graph)->graph.concat(static_cast(a), static_cast(b), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_cat(cactus_graph_t graph, const cactus_node_t* nodes, size_t count, int32_t axis, cactus_node_t* out) { - if (!graph || !nodes || !out || count == 0) { - last_error_message = "Invalid args to cactus_graph_cat"; - return -1; - } - try { - std::vector ns; - for (size_t i = 0; i < count; ++i) { - ns.push_back(static_cast(nodes[i])); - } - *out = static_cast(as_graph(graph)->graph.cat(ns, axis)); - return 0; - - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_matmul(cactus_graph_t graph, cactus_node_t a, cactus_node_t b, bool pretransposed_rhs, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_matmul"); - try { - *out = static_cast(as_graph(graph)->graph.matmul(static_cast(a), static_cast(b), pretransposed_rhs, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gather(cactus_graph_t graph, cactus_node_t tensor, cactus_node_t indices, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gather"); - try { - *out = static_cast(as_graph(graph)->graph.gather(static_cast(tensor), static_cast(indices))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_embedding_from_tensor(cactus_graph_t graph, cactus_node_t embedding_tensor, cactus_node_t indices, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_tensor"); - try { - *out = static_cast(as_graph(graph)->graph.embedding(static_cast(embedding_tensor), static_cast(indices))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_embedding_from_file(cactus_graph_t graph, const char* filename, cactus_node_t indices, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_embedding_from_file"); - try { - *out = static_cast(as_graph(graph)->graph.embedding(std::string(filename), static_cast(indices))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mmap_embeddings(cactus_graph_t graph, const char* filename, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_embeddings"); - try { - *out = static_cast(as_graph(graph)->graph.mmap_embeddings(std::string(filename))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_mmap_weights(cactus_graph_t graph, const char* filename, cactus_node_t* out) { - if (!graph || !filename || !out) return fail_invalid("Invalid args to cactus_graph_mmap_weights"); - try { - *out = static_cast(as_graph(graph)->graph.mmap_weights(std::string(filename))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_bilinear_interpolation(cactus_graph_t graph, cactus_node_t pos_embeds, size_t dst_height, size_t dst_width, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_bilinear_interpolation"); - try { - *out = static_cast(as_graph(graph)->graph.bilinear_interpolation(static_cast(pos_embeds), dst_height, dst_width)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_grouped_scales(cactus_graph_t graph, cactus_node_t node, size_t group_size, size_t num_groups, void* scales_ptr) { - if (!graph || !scales_ptr) return fail_invalid("Invalid args to cactus_graph_set_grouped_scales"); - try { - as_graph(graph)->graph.set_grouped_scales(static_cast(node), group_size, num_groups, scales_ptr); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_set_interleaved(cactus_graph_t graph, cactus_node_t node, bool interleaved, size_t original_n) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_set_interleaved"); - try { - as_graph(graph)->graph.set_interleaved(static_cast(node), interleaved, original_n); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_release_weight_pages(cactus_graph_t graph, cactus_node_t node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_release_weight_pages"); - try { - as_graph(graph)->graph.release_weight_pages(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_prefetch_weight_pages(cactus_graph_t graph, cactus_node_t node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_prefetch_weight_pages"); - try { - as_graph(graph)->graph.prefetch_weight_pages(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_release_all_weight_pages(cactus_graph_t graph) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_release_all_weight_pages"); - try { - as_graph(graph)->graph.release_all_weight_pages(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_relu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_relu"); - try { - *out = static_cast(as_graph(graph)->graph.relu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_silu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_silu"); - try { - *out = static_cast(as_graph(graph)->graph.silu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gelu(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu"); - try { - *out = static_cast(as_graph(graph)->graph.gelu(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gelu_erf(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gelu_erf"); - try { - *out = static_cast(as_graph(graph)->graph.gelu_erf(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sigmoid(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sigmoid"); - try { - *out = static_cast(as_graph(graph)->graph.sigmoid(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_tanh(cactus_graph_t graph, cactus_node_t x, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_tanh"); - try { - *out = static_cast(as_graph(graph)->graph.tanh(static_cast(x))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_glu(cactus_graph_t graph, cactus_node_t x, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_glu"); - try { - *out = static_cast(as_graph(graph)->graph.glu(static_cast(x), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_layernorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, float epsilon, bool has_bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_layernorm"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), static_cast(bias), epsilon)); - } else { - *out = static_cast(as_graph(graph)->graph.layernorm(static_cast(input), static_cast(weight), epsilon)); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_groupnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, size_t num_groups, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_groupnorm"); - try { - *out = static_cast(as_graph(graph)->graph.groupnorm(static_cast(input), static_cast(weight), static_cast(bias), num_groups, epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_batchnorm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t running_mean, cactus_node_t running_var, int32_t axis, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_batchnorm"); - try { - *out = static_cast(as_graph(graph)->graph.batchnorm(static_cast(input), static_cast(weight), static_cast(bias), static_cast(running_mean), static_cast(running_var), axis, epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_topk(cactus_graph_t graph, cactus_node_t input, size_t k, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_topk"); - try { - *out = static_cast(as_graph(graph)->graph.topk(static_cast(input), k)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rms_norm(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, float epsilon, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rms_norm"); - try { - *out = static_cast(as_graph(graph)->graph.rms_norm(static_cast(input), static_cast(weight), epsilon)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rope(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope"); - try { - *out = static_cast(as_graph(graph)->graph.rope(static_cast(input), theta, position_offset, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rope_gptj(cactus_graph_t graph, cactus_node_t input, float theta, size_t position_offset, size_t rot_dim, int32_t backend, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rope_gptj"); - try { - *out = static_cast(as_graph(graph)->graph.rope_gptj(static_cast(input), theta, position_offset, rot_dim, static_cast(backend))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_softmax(cactus_graph_t graph, cactus_node_t input, int32_t axis, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_softmax"); - try { - *out = static_cast(as_graph(graph)->graph.softmax(static_cast(input), axis)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_attention(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, float scale, bool is_causal, size_t position_offset, size_t window_size, int32_t backend, bool use_mask, cactus_node_t mask, bool additive_mask, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention"); - try { - if (use_mask) { - *out = static_cast(as_graph(graph)->graph.attention_masked(static_cast(query), static_cast(key), static_cast(value), static_cast(mask), scale, is_causal, static_cast(backend), additive_mask, position_offset, window_size)); - } else if (window_size > 0 || position_offset > 0) { - *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, position_offset, window_size, static_cast(backend))); - } else { - *out = static_cast(as_graph(graph)->graph.attention(static_cast(query), static_cast(key), static_cast(value), scale, is_causal, static_cast(backend))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_rel_pos_bias(cactus_graph_t graph, cactus_node_t query, cactus_node_t relative_key, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_rel_pos_bias"); - try { - *out = static_cast(as_graph(graph)->graph.rel_pos_bias(static_cast(query), static_cast(relative_key), scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_attention_int8_hybrid(cactus_graph_t graph, cactus_node_t query, cactus_node_t key_new, cactus_node_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, size_t window_size, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_attention_int8_hybrid"); - try { - *out = static_cast(as_graph(graph)->graph.attention_int8_hybrid(static_cast(query), static_cast(key_new), static_cast(value_new), scale, position_offset, cached_keys, cached_values, k_scales, v_scales, cache_len, num_kv_heads, head_dim, window_size)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_causal(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t kernel_size, size_t dilation, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_causal"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_causal(static_cast(input), static_cast(weight), kernel_size, dilation)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_k3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k3"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_k3(static_cast(input), static_cast(weight), stride)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_k7s3(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_k7s3"); - try { - *out = static_cast(as_graph(graph)->graph.conv1d_k7s3(static_cast(input), static_cast(weight), static_cast(bias))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, size_t stride, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), static_cast(bias), stride)); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d(static_cast(input), static_cast(weight), stride)); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_same_depthwise_k9(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_same_depthwise_k9"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d_same_depthwise_k9(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv1d_pointwise(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv1d_pointwise"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv1d_pointwise(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_k3s2p1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_k3s2p1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_depthwise_k3s2p1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_depthwise_k3s2p1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_depthwise_k3s2p1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_conv2d_pointwise_1x1(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, bool has_bias, cactus_node_t bias, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_conv2d_pointwise_1x1"); - try { - if (has_bias) { - *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight), static_cast(bias))); - } else { - *out = static_cast(as_graph(graph)->graph.conv2d_pointwise_1x1(static_cast(input), static_cast(weight))); - } - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_lstm_cell(cactus_graph_t graph, cactus_node_t input, cactus_node_t h_prev, cactus_node_t c_prev, cactus_node_t weight_ih, cactus_node_t weight_hh, cactus_node_t bias_ih, cactus_node_t bias_hh, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_lstm_cell"); - try { - *out = static_cast(as_graph(graph)->graph.lstm_cell(static_cast(input), static_cast(h_prev), static_cast(c_prev), static_cast(weight_ih), static_cast(weight_hh), static_cast(bias_ih), static_cast(bias_hh))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gated_deltanet_decode(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_decode"); - try { - *out = static_cast(as_graph(graph)->graph.gated_deltanet_decode(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gated_deltanet_prefill(cactus_graph_t graph, cactus_node_t query, cactus_node_t key, cactus_node_t value, cactus_node_t gate_log, cactus_node_t beta, cactus_node_t initial_state, size_t chunk_size, float scale, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gated_deltanet_prefill"); - try { - *out = static_cast(as_graph(graph)->graph.gated_deltanet_prefill(static_cast(query), static_cast(key), static_cast(value), static_cast(gate_log), static_cast(beta), static_cast(initial_state), chunk_size, scale)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_stft(cactus_graph_t graph, cactus_node_t input, cactus_node_t weight, size_t stride, size_t num_fft_bins, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_stft"); - try { - *out = static_cast(as_graph(graph)->graph.stft(static_cast(input), static_cast(weight), stride, num_fft_bins)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_altup_predict(cactus_graph_t graph, cactus_node_t coefs, const cactus_node_t* streams, size_t num_streams, cactus_node_t* out) { - if (!graph || !streams || num_streams == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_predict"); - try { - std::vector s(num_streams); - for (size_t i = 0; i < num_streams; ++i) s[i] = static_cast(streams[i]); - *out = static_cast(as_graph(graph)->graph.altup_predict(static_cast(coefs), s.data(), num_streams)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_altup_correct(cactus_graph_t graph, cactus_node_t coefs, cactus_node_t innovation, const cactus_node_t* predictions, size_t num_predictions, cactus_node_t* out) { - if (!graph || !predictions || num_predictions == 0 || !out) return fail_invalid("Invalid args to cactus_graph_altup_correct"); - try { - std::vector p(num_predictions); - for (size_t i = 0; i < num_predictions; ++i) p[i] = static_cast(predictions[i]); - *out = static_cast(as_graph(graph)->graph.altup_correct(static_cast(coefs), static_cast(innovation), p.data(), num_predictions)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_gaussian_topk(cactus_graph_t graph, cactus_node_t input, float ppf, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_gaussian_topk"); - try { - *out = static_cast(as_graph(graph)->graph.gaussian_topk(static_cast(input), ppf)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_moe_layer_gated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w3_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, cactus_node_t* out) { - if (!graph || !w1_weights || !w3_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_gated"); - try { - std::vector w1(num_experts), w3(num_experts), w2(num_experts); - for (size_t i = 0; i < num_experts; ++i) { - w1[i] = static_cast(w1_weights[i]); - w3[i] = static_cast(w3_weights[i]); - w2[i] = static_cast(w2_weights[i]); - } - *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w3, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_moe_layer_ungated(cactus_graph_t graph, cactus_node_t hidden, cactus_node_t routing_probs, cactus_node_t topk_indices, - const cactus_node_t* w1_weights, const cactus_node_t* w2_weights, - size_t num_experts, size_t num_experts_per_tok, bool normalize_routing, float epsilon, float routed_scaling_factor, int32_t activation, cactus_node_t* out) { - if (!graph || !w1_weights || !w2_weights || !out) return fail_invalid("Invalid args to cactus_graph_moe_layer_ungated"); - try { - std::vector w1(num_experts), w2(num_experts); - for (size_t i = 0; i < num_experts; ++i) { - w1[i] = static_cast(w1_weights[i]); - w2[i] = static_cast(w2_weights[i]); - } - *out = static_cast(as_graph(graph)->graph.moe_layer(static_cast(hidden), static_cast(routing_probs), static_cast(topk_indices), w1, w2, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, static_cast(activation))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_sample(cactus_graph_t graph, cactus_node_t logits, float temperature, float top_p, size_t top_k, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_sample"); - try { - std::unordered_map empty_bias; - *out = static_cast(as_graph(graph)->graph.sample(static_cast(logits), temperature, top_p, top_k, empty_bias)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_scatter_topk(cactus_graph_t graph, cactus_node_t indices, cactus_node_t values, size_t num_classes, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_scatter_topk"); - try { - *out = static_cast(as_graph(graph)->graph.scatter_topk(static_cast(indices), static_cast(values), num_classes)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_persistent(cactus_graph_t graph, cactus_node_t source_node, cactus_node_t* out) { - if (!graph || !out) return fail_invalid("Invalid args to cactus_graph_persistent"); - try { - *out = static_cast(as_graph(graph)->graph.persistent(static_cast(source_node))); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_is_populated(cactus_graph_t graph, cactus_node_t persistent_node, int32_t* out_is_populated) { - if (!graph || !out_is_populated) return fail_invalid("Invalid args to cactus_graph_is_populated"); - try { - *out_is_populated = as_graph(graph)->graph.is_populated(static_cast(persistent_node)) ? 1 : 0; - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_invalidate_persistent(cactus_graph_t graph, cactus_node_t persistent_node) { - if (!graph) return fail_invalid("Invalid args to cactus_graph_invalidate_persistent"); - try { - as_graph(graph)->graph.invalidate_persistent(static_cast(persistent_node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_execute(cactus_graph_t graph) { - if (!graph) { - last_error_message = "Graph is null"; - return -1; - } - try { - as_graph(graph)->graph.execute(); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_get_output_ptr(cactus_graph_t graph, cactus_node_t node, void** out_ptr) { - if (!graph || !out_ptr) { - last_error_message = "Invalid args to cactus_graph_get_output_ptr"; - return -1; - } - try { - *out_ptr = as_graph(graph)->graph.get_output(static_cast(node)); - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} - -int cactus_graph_get_output_info(cactus_graph_t graph, cactus_node_t node, cactus_tensor_info_t* out_info) { - if (!graph || !out_info) { - last_error_message = "Invalid args to cactus_graph_get_output_info"; - return -1; - } - try { - const auto& buf = as_graph(graph)->graph.get_output_buffer(static_cast(node)); - out_info->precision = static_cast(buf.precision); - out_info->rank = buf.shape.size(); - if (out_info->rank > 8) { - last_error_message = "Rank exceeds cactus_tensor_info_t shape capacity"; - return -1; - } - for (size_t i = 0; i < out_info->rank; ++i) { - out_info->shape[i] = buf.shape[i]; - } - for (size_t i = out_info->rank; i < 8; ++i) { - out_info->shape[i] = 0; - } - out_info->num_elements = buf.total_size; - out_info->byte_size = buf.byte_size; - return 0; - } catch (const std::exception& e) { - last_error_message = e.what(); - return -1; - } -} -} diff --git a/cactus/ffi/cactus_index.cpp b/cactus/ffi/cactus_index.cpp deleted file mode 100644 index a81f62377..000000000 --- a/cactus/ffi/cactus_index.cpp +++ /dev/null @@ -1,458 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include - -using namespace cactus::ffi; - -struct CactusIndexHandle { - std::unique_ptr index; -}; - -static cactus::engine::index::QueryOptions parse_query_options_json(const std::string& json) { - cactus::engine::index::QueryOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"score_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.score_threshold = std::stof(json.substr(pos)); - } - - return options; -} - -extern "C" { - -cactus_index_t cactus_index_init(const char* index_dir, size_t embedding_dim) { - if (!index_dir) { - last_error_message = "Index directory path cannot be null"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - if (embedding_dim == 0) { - last_error_message = "Embedding dimension must be greater than 0"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - std::string dir_path(index_dir); - if (dir_path.empty()) { - last_error_message = "Index directory path cannot be empty"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } - - std::string index_path_str = dir_path + "/index.bin"; - std::string data_path_str = dir_path + "/data.bin"; - - CACTUS_LOG_INFO("index_init", "Initializing index in directory: " << dir_path << ", dim: " << embedding_dim); - - try { - auto* handle = new CactusIndexHandle(); - handle->index = std::make_unique( - index_path_str, - data_path_str, - embedding_dim - ); - - if (!handle->index) { - last_error_message = "Failed to create index instance"; - CACTUS_LOG_ERROR("index_init", last_error_message); - - delete handle; - return nullptr; - } - - CACTUS_LOG_INFO("index_init", "Index initialized successfully"); - return handle; - - } catch (const std::exception& e) { - last_error_message = "Exception during index init: " + std::string(e.what()); - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during index initialization"; - CACTUS_LOG_ERROR("index_init", last_error_message); - return nullptr; - } -} - -int cactus_index_add( - cactus_index_t index, - const int* ids, - const char** documents, - const char** metadatas, - const float** embeddings, - size_t count, - size_t embedding_dim -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (!ids || !documents || !embeddings) { - last_error_message = "Invalid parameters: ids, documents, or embeddings is null"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (count == 0) { - last_error_message = "Invalid parameter: count must be greater than 0"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - if (embedding_dim == 0) { - last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector docs; - docs.reserve(count); - - for (size_t i = 0; i < count; ++i) { - if (!embeddings[i]) { - last_error_message = "Embedding at index " + std::to_string(i) + " is null"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } - - docs.emplace_back( - ids[i], - std::vector(embeddings[i], embeddings[i] + embedding_dim), - documents[i] ? std::string(documents[i]) : "", - metadatas && metadatas[i] ? std::string(metadatas[i]) : "" - ); - } - - CACTUS_LOG_INFO("index_add", "Adding " << docs.size() << " documents"); - handle->index->add_documents(docs); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_add", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during add"; - CACTUS_LOG_ERROR("index_add", last_error_message); - return -1; - } -} - -int cactus_index_delete( - cactus_index_t index, - const int* ids, - size_t ids_count -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } - - if (!ids || ids_count == 0) { - last_error_message = "Invalid parameters"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector doc_ids(ids, ids + ids_count); - - CACTUS_LOG_INFO("index_delete", "Deleting " << doc_ids.size() << " documents"); - handle->index->delete_documents(doc_ids); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_delete", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during delete"; - CACTUS_LOG_ERROR("index_delete", last_error_message); - return -1; - } -} - -int cactus_index_get( - cactus_index_t index, - const int* ids, - size_t ids_count, - char** document_buffers, - size_t* document_buffer_sizes, - char** metadata_buffers, - size_t* metadata_buffer_sizes, - float** embedding_buffers, - size_t* embedding_buffer_sizes -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (!ids) { - last_error_message = "Invalid parameter: ids is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (ids_count == 0) { - last_error_message = "Invalid parameter: ids_count must be greater than 0"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (document_buffers && !document_buffer_sizes) { - last_error_message = "Invalid parameters: document_buffers provided but document_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (metadata_buffers && !metadata_buffer_sizes) { - last_error_message = "Invalid parameters: metadata_buffers provided but metadata_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - if (embedding_buffers && !embedding_buffer_sizes) { - last_error_message = "Invalid parameters: embedding_buffers provided but embedding_buffer_sizes is null"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector doc_ids(ids, ids + ids_count); - - CACTUS_LOG_INFO("index_get", "Getting " << doc_ids.size() << " documents"); - std::vector docs = handle->index->get_documents(doc_ids); - - for (size_t i = 0; i < docs.size(); ++i) { - if (document_buffers) { - size_t document_size = docs[i].content.size() + 1; - if (document_buffer_sizes[i] < document_size) { - last_error_message = "Document buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(document_size) + - ", provided: " + std::to_string(document_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - - if (metadata_buffers) { - size_t metadata_size = docs[i].metadata.size() + 1; - if (metadata_buffer_sizes[i] < metadata_size) { - last_error_message = "Metadata buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(metadata_size) + - ", provided: " + std::to_string(metadata_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - - if (embedding_buffers) { - size_t embedding_size = docs[i].embedding.size(); - if (embedding_buffer_sizes[i] < embedding_size) { - last_error_message = "Embedding buffer too small at index " + std::to_string(i) + - " (required: " + std::to_string(embedding_size) + - ", provided: " + std::to_string(embedding_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } - } - } - - for (size_t i = 0; i < docs.size(); ++i) { - if (document_buffers) { - size_t document_size = docs[i].content.size() + 1; - std::memcpy(document_buffers[i], docs[i].content.c_str(), document_size); - document_buffer_sizes[i] = document_size; - } - - if (metadata_buffers) { - size_t metadata_size = docs[i].metadata.size() + 1; - std::memcpy(metadata_buffers[i], docs[i].metadata.c_str(), metadata_size); - metadata_buffer_sizes[i] = metadata_size; - } - - if (embedding_buffers) { - size_t embedding_size = docs[i].embedding.size(); - std::memcpy(embedding_buffers[i], docs[i].embedding.data(), embedding_size * sizeof(float)); - embedding_buffer_sizes[i] = embedding_size; - } - } - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_get", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during get"; - CACTUS_LOG_ERROR("index_get", last_error_message); - return -1; - } -} - -int cactus_index_query( - cactus_index_t index, - const float** embeddings, - size_t embeddings_count, - size_t embedding_dim, - const char* options_json, - int** id_buffers, - size_t* id_buffer_sizes, - float** score_buffers, - size_t* score_buffer_sizes -) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!embeddings) { - last_error_message = "Invalid parameter: embeddings is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!id_buffers || !score_buffers) { - last_error_message = "Invalid parameters: id_buffers or score_buffers is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (!id_buffer_sizes || !score_buffer_sizes) { - last_error_message = "Invalid parameters: buffer size arrays cannot be null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (embeddings_count == 0) { - last_error_message = "Invalid parameter: embeddings_count must be greater than 0"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - if (embedding_dim == 0) { - last_error_message = "Invalid parameter: embedding_dim must be greater than 0"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - std::vector> embeddings_vec; - embeddings_vec.reserve(embeddings_count); - - for (size_t i = 0; i < embeddings_count; ++i) { - if (!embeddings[i]) { - last_error_message = "Embedding at index " + std::to_string(i) + " is null"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - embeddings_vec.emplace_back(embeddings[i], embeddings[i] + embedding_dim); - } - - cactus::engine::index::QueryOptions options; - if (options_json && std::strlen(options_json) > 0) { - options = parse_query_options_json(options_json); - } - - CACTUS_LOG_DEBUG("index_query", "Querying with " << embeddings_count << " queries, dim: " << embedding_dim << ", top_k: " << options.top_k); - std::vector> results = handle->index->query(embeddings_vec, options); - - for (size_t i = 0; i < results.size(); ++i) { - size_t result_count = results[i].size(); - if (id_buffer_sizes[i] < result_count || score_buffer_sizes[i] < result_count) { - last_error_message = "Result buffer too small at query index " + std::to_string(i) + - " (required: " + std::to_string(result_count) + - ", id_buffer: " + std::to_string(id_buffer_sizes[i]) + - ", score_buffer: " + std::to_string(score_buffer_sizes[i]) + ")"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } - } - - for (size_t i = 0; i < results.size(); ++i) { - size_t result_count = results[i].size(); - - for (size_t j = 0; j < result_count; ++j) { - id_buffers[i][j] = results[i][j].doc_id; - score_buffers[i][j] = results[i][j].score; - } - - id_buffer_sizes[i] = result_count; - score_buffer_sizes[i] = result_count; - } - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_query", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during query"; - CACTUS_LOG_ERROR("index_query", last_error_message); - return -1; - } -} - -int cactus_index_compact(cactus_index_t index) { - if (!index) { - last_error_message = "Index not initialized"; - CACTUS_LOG_ERROR("index_compact", last_error_message); - return -1; - } - - try { - auto* handle = static_cast(index); - - CACTUS_LOG_INFO("index_compact", "Compacting index"); - handle->index->compact(); - - return 0; - - } catch (const std::exception& e) { - last_error_message = std::string(e.what()); - CACTUS_LOG_ERROR("index_compact", "Exception: " << e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown error during compact"; - CACTUS_LOG_ERROR("index_compact", last_error_message); - return -1; - } -} - -void cactus_index_destroy(cactus_index_t index) { - if (index) delete static_cast(index); -} - -} diff --git a/cactus/ffi/cactus_init.cpp b/cactus/ffi/cactus_init.cpp deleted file mode 100644 index d3e4fd122..000000000 --- a/cactus/ffi/cactus_init.cpp +++ /dev/null @@ -1,485 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t RAG_MAX_CHUNK_TOKENS = 128; -static constexpr size_t RAG_MIN_CHUNK_TOKENS = 24; -static constexpr size_t RAG_CHUNK_OVERLAP = 32; - -static void apply_no_cloud_telemetry_env() { - if (cactus::ffi::env_flag_enabled("CACTUS_NO_CLOUD_TELE")) { - cactus::telemetry::setCloudDisabled(true); - } -} - -static time_t get_file_mtime(const std::string& path) { - struct stat st; - if (stat(path.c_str(), &st) == 0) { - return st.st_mtime; - } - return 0; -} - -static bool corpus_is_stale(const std::string& corpus_dir) { - std::string index_path = corpus_dir + "/index.bin"; - time_t index_mtime = get_file_mtime(index_path); - if (index_mtime == 0) return true; - - DIR* dir = opendir(corpus_dir.c_str()); - if (!dir) return true; - - struct dirent* entry; - while ((entry = readdir(dir)) != nullptr) { - std::string name = entry->d_name; - if (name == "." || name == "..") continue; - if (name == "index.bin" || name == "data.bin") continue; - - bool is_corpus_file = false; - if (name.size() > 4 && name.substr(name.size() - 4) == ".txt") is_corpus_file = true; - if (name.size() > 3 && name.substr(name.size() - 3) == ".md") is_corpus_file = true; - - if (is_corpus_file) { - std::string full_path = corpus_dir + "/" + name; - time_t file_mtime = get_file_mtime(full_path); - if (file_mtime > index_mtime) { - closedir(dir); - CACTUS_LOG_INFO("init", "Corpus file " << name << " is newer than index, rebuilding"); - return true; - } - } - } - closedir(dir); - return false; -} - -static std::string read_file_contents(const std::string& path) { - std::ifstream file(path); - if (!file.is_open()) return ""; - std::stringstream buffer; - buffer << file.rdbuf(); - return buffer.str(); -} - -static std::vector scan_corpus_files(const std::string& corpus_dir) { - std::vector files; - DIR* dir = opendir(corpus_dir.c_str()); - if (!dir) return files; - - struct dirent* entry; - while ((entry = readdir(dir)) != nullptr) { - std::string name = entry->d_name; - if (name == "." || name == "..") continue; - if (name == "index.bin" || name == "data.bin") continue; - - std::string full_path = corpus_dir + "/" + name; - struct stat st; - if (stat(full_path.c_str(), &st) == 0 && S_ISREG(st.st_mode)) { - if (name.size() > 4 && (name.substr(name.size() - 4) == ".txt" || name.substr(name.size() - 3) == ".md")) { - files.push_back(full_path); - } - } - } - closedir(dir); - return files; -} - -static std::vector split_into_paragraphs(const std::string& content) { - std::vector paragraphs; - std::string current; - - size_t i = 0; - while (i < content.size()) { - // Check for markdown header - if (content[i] == '#' && (i == 0 || content[i-1] == '\n')) { - if (!current.empty()) { - paragraphs.push_back(current); - current.clear(); - } - // Include the header line - while (i < content.size() && content[i] != '\n') { - current += content[i++]; - } - if (i < content.size()) current += content[i++]; - continue; - } - - // Check for double newline (paragraph break) - if (content[i] == '\n' && i + 1 < content.size() && content[i+1] == '\n') { - current += content[i]; - if (!current.empty() && current != "\n") { - paragraphs.push_back(current); - current.clear(); - } - i++; - // Skip multiple blank lines - while (i < content.size() && content[i] == '\n') i++; - continue; - } - - current += content[i++]; - } - - if (!current.empty()) { - paragraphs.push_back(current); - } - - return paragraphs; -} - -static std::vector> chunk_corpus( - const std::vector& file_paths, - Tokenizer* tokenizer -) { - std::vector> chunks; - - for (const auto& path : file_paths) { - std::string content = read_file_contents(path); - if (content.empty()) continue; - - std::string filename = path; - size_t last_slash = path.find_last_of("/\\"); - if (last_slash != std::string::npos) { - filename = path.substr(last_slash + 1); - } - - auto paragraphs = split_into_paragraphs(content); - - std::string current_chunk; - size_t current_tokens = 0; - - for (const auto& para : paragraphs) { - std::vector para_tokens = tokenizer->encode(para); - size_t para_token_count = para_tokens.size(); - - // If paragraph alone is too large, split it by tokens - if (para_token_count > RAG_MAX_CHUNK_TOKENS) { - // Flush current chunk first - if (!current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - current_chunk.clear(); - current_tokens = 0; - } - - // Split large paragraph by sentences or fixed tokens - size_t stride = RAG_MAX_CHUNK_TOKENS - RAG_CHUNK_OVERLAP; - for (size_t i = 0; i < para_tokens.size(); i += stride) { - size_t end = std::min(i + RAG_MAX_CHUNK_TOKENS, para_tokens.size()); - std::vector chunk_tokens(para_tokens.begin() + i, para_tokens.begin() + end); - std::string chunk_text = tokenizer->decode(chunk_tokens); - chunks.emplace_back(chunk_text, filename); - if (end >= para_tokens.size()) break; - } - continue; - } - - // Would adding this paragraph exceed max? - if (current_tokens + para_token_count > RAG_MAX_CHUNK_TOKENS && !current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - current_chunk.clear(); - current_tokens = 0; - } - - // Add paragraph to current chunk - if (!current_chunk.empty()) current_chunk += "\n"; - current_chunk += para; - current_tokens += para_token_count; - } - - // Don't forget the last chunk - if (!current_chunk.empty() && current_tokens >= RAG_MIN_CHUNK_TOKENS) { - chunks.emplace_back(current_chunk, filename); - } else if (!current_chunk.empty() && !chunks.empty()) { - // Append small remaining chunk to previous - chunks.back().first += "\n" + current_chunk; - } else if (!current_chunk.empty()) { - chunks.emplace_back(current_chunk, filename); - } - } - - return chunks; -} - -static bool build_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { - CACTUS_LOG_INFO("init", "Building corpus index from: " << corpus_dir); - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("init", "No tokenizer available for corpus indexing"); - return false; - } - - auto file_paths = scan_corpus_files(corpus_dir); - if (file_paths.empty()) { - CACTUS_LOG_WARN("init", "No .txt or .md files found in corpus directory"); - return false; - } - - CACTUS_LOG_INFO("init", "Found " << file_paths.size() << " corpus files"); - - auto chunks = chunk_corpus(file_paths, tokenizer); - if (chunks.empty()) { - CACTUS_LOG_WARN("init", "No chunks generated from corpus"); - return false; - } - - CACTUS_LOG_INFO("init", "Generated " << chunks.size() << " chunks from corpus"); - - std::vector test_tokens = tokenizer->encode("test"); - std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); - if (test_embedding.empty()) { - CACTUS_LOG_ERROR("init", "Failed to get embedding dimension"); - return false; - } - size_t embedding_dim = test_embedding.size(); - handle->corpus_embedding_dim = embedding_dim; - - CACTUS_LOG_INFO("init", "Embedding dimension: " << embedding_dim); - - std::string index_path = corpus_dir + "/index.bin"; - std::string data_path = corpus_dir + "/data.bin"; - - std::remove(index_path.c_str()); - std::remove(data_path.c_str()); - - try { - handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("init", "Failed to create index: " << e.what()); - return false; - } - - std::vector docs; - docs.reserve(chunks.size()); - - for (size_t i = 0; i < chunks.size(); ++i) { - const auto& [chunk_text, source_file] = chunks[i]; - - std::vector tokens = tokenizer->encode(chunk_text); - std::vector embedding = handle->model->get_embeddings(tokens, true, true); - - if (embedding.size() != embedding_dim) { - CACTUS_LOG_WARN("init", "Skipping chunk " << i << " - embedding dimension mismatch"); - continue; - } - - docs.push_back(index::Document{ - static_cast(i), - std::move(embedding), - chunk_text, - source_file - }); - - if ((i + 1) % 50 == 0) { - CACTUS_LOG_INFO("init", "Embedded " << (i + 1) << "/" << chunks.size() << " chunks"); - } - } - - if (docs.empty()) { - CACTUS_LOG_ERROR("init", "No documents to add to index"); - return false; - } - - try { - handle->corpus_index->add_documents(docs); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("init", "Failed to add documents to index: " << e.what()); - return false; - } - - CACTUS_LOG_INFO("init", "Corpus index built successfully with " << docs.size() << " chunks"); - return true; -} - -static bool load_corpus_index(CactusModelHandle* handle, const std::string& corpus_dir) { - std::string index_path = corpus_dir + "/index.bin"; - std::string data_path = corpus_dir + "/data.bin"; - - struct stat st; - if (stat(index_path.c_str(), &st) != 0 || stat(data_path.c_str(), &st) != 0) { - return false; - } - - if (corpus_is_stale(corpus_dir)) { - return false; - } - - auto* tokenizer = handle->model->get_tokenizer(); - std::vector test_tokens = tokenizer->encode("test"); - std::vector test_embedding = handle->model->get_embeddings(test_tokens, true, true); - if (test_embedding.empty()) { - CACTUS_LOG_ERROR("init", "Failed to get embedding dimension for index loading"); - return false; - } - size_t embedding_dim = test_embedding.size(); - handle->corpus_embedding_dim = embedding_dim; - - try { - handle->corpus_index = std::make_unique(index_path, data_path, embedding_dim); - CACTUS_LOG_INFO("init", "Loaded existing corpus index from: " << corpus_dir); - return true; - } catch (const std::exception& e) { - CACTUS_LOG_WARN("init", "Failed to load existing index: " << e.what()); - return false; - } -} - -std::string last_error_message; - -bool matches_stop_sequence(const std::vector& generated_tokens, - const std::vector>& stop_sequences) { - for (const auto& stop_seq : stop_sequences) { - if (stop_seq.empty()) continue; - if (generated_tokens.size() >= stop_seq.size()) { - if (std::equal(stop_seq.rbegin(), stop_seq.rend(), generated_tokens.rbegin())) - return true; - } - } - return false; -} - -extern "C" { - -const char* cactus_get_last_error() { - return last_error_message.c_str(); -} - -cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index) { - constexpr size_t DEFAULT_CONTEXT_SIZE = 512; // matches default sliding window size - - std::string model_path_str = model_path ? std::string(model_path) : "unknown"; - - std::string model_name = model_path_str; - size_t last_slash = model_path_str.find_last_of("/\\"); - if (last_slash != std::string::npos) { - model_name = model_path_str.substr(last_slash + 1); - } - - CACTUS_LOG_INFO("init", "Loading model: " << model_name << " from " << model_path_str); - - apply_no_cloud_telemetry_env(); - cactus::telemetry::init(nullptr, model_path_str.c_str(), nullptr); - - auto __cactus_init_start = std::chrono::steady_clock::now(); - - try { - auto* handle = new CactusModelHandle(); - handle->model = create_model(model_path); - handle->model_name = model_name; - - if (!handle->model) { - last_error_message = "Failed to create model - check config.txt exists at: " + model_path_str; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - delete handle; - return nullptr; - } - - if (!handle->model->init(model_path, DEFAULT_CONTEXT_SIZE)) { - last_error_message = "Failed to initialize model - check weight files at: " + model_path_str; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - delete handle; - return nullptr; - } - - auto model_type = handle->model->get_config().model_type; - if (model_type == Config::ModelType::WHISPER || - model_type == Config::ModelType::MOONSHINE || - model_type == Config::ModelType::PARAKEET || - model_type == Config::ModelType::PARAKEET_TDT) { - std::string vad_path = model_path_str + "/vad"; - handle->vad_model = create_model(vad_path); - if (!handle->vad_model) { - last_error_message = "Failed to create VAD model - check VAD weights at: " + vad_path; - CACTUS_LOG_ERROR("init", last_error_message); - delete handle; - return nullptr; - } - if (!handle->vad_model->init(vad_path, 0, "", false)) { - last_error_message = "Failed to initialize VAD model - check VAD weight files at: " + vad_path; - CACTUS_LOG_ERROR("init", last_error_message); - delete handle; - return nullptr; - } - } - - if (corpus_dir != nullptr && strlen(corpus_dir) > 0) { - handle->corpus_dir = std::string(corpus_dir); - - bool loaded = false; - if (cache_index) { - loaded = load_corpus_index(handle, handle->corpus_dir); - } - - if (!loaded) { - CACTUS_LOG_INFO("init", (cache_index ? "No existing index found, building new corpus index" : "Building fresh corpus index (caching disabled)")); - if (!build_corpus_index(handle, handle->corpus_dir)) { - CACTUS_LOG_WARN("init", "Failed to build corpus index - RAG disabled"); - } - } - } - - CACTUS_LOG_INFO("init", "Model loaded successfully: " << model_name); - { - auto __cactus_init_ok_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), true, static_cast(__cactus_init_ok_dur), ""); - } - - return handle; - } catch (const std::exception& e) { - last_error_message = "Exception during init: " + std::string(e.what()); - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during model initialization"; - CACTUS_LOG_ERROR("init", last_error_message); - { - auto __cactus_init_err_dur = std::chrono::duration_cast(std::chrono::steady_clock::now() - __cactus_init_start).count(); - cactus::telemetry::recordInit(model_name.c_str(), false, static_cast(__cactus_init_err_dur), last_error_message.c_str()); - } - return nullptr; - } -} - -void cactus_destroy(cactus_model_t model) { - if (model) delete static_cast(model); -} - -void cactus_reset(cactus_model_t model) { - if (!model) return; - auto* handle = static_cast(model); - handle->model->reset_cache(); - handle->processed_tokens.clear(); - handle->processed_images.clear(); - handle->user_audio_counts.clear(); -} - -void cactus_stop(cactus_model_t model) { - if (!model) return; - auto* handle = static_cast(model); - handle->should_stop = true; -} - -} diff --git a/cactus/ffi/cactus_log.cpp b/cactus/ffi/cactus_log.cpp deleted file mode 100644 index d067f9adf..000000000 --- a/cactus/ffi/cactus_log.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "cactus_ffi.h" -#include "graph/graph.h" - -extern "C" { - -void cactus_log_set_level(int level) { - cactus::Logger::instance().set_level(static_cast(level)); -} - -void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data) { - if (callback) { - cactus::Logger::instance().set_callback( - [callback, user_data](cactus::LogLevel level, - const std::string& component, - const std::string& message) { - callback(static_cast(level), component.c_str(), message.c_str(), user_data); - }); - } else { - cactus::Logger::instance().set_callback(nullptr); - } -} - -} diff --git a/cactus/ffi/cactus_rag.cpp b/cactus/ffi/cactus_rag.cpp deleted file mode 100644 index a0765f787..000000000 --- a/cactus/ffi/cactus_rag.cpp +++ /dev/null @@ -1,459 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - -static constexpr size_t RAG_TOP_K = 5; -static constexpr size_t RAG_CANDIDATE_K = 20; -static constexpr float RRF_K = 60.0f; -static constexpr float RRF_EMB_WEIGHT = 0.8f; -static constexpr float RRF_BM25_WEIGHT = 0.2f; -static constexpr float BM25_K1 = 1.5f; -static constexpr float BM25_B = 0.75f; - -static std::vector> compute_rrf_scores( - const std::vector>& emb_ranked, - const std::vector>& bm25_ranked, - size_t num_items -) { - std::vector> emb_sorted = emb_ranked; - std::vector> bm25_sorted = bm25_ranked; - - std::sort(emb_sorted.begin(), emb_sorted.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - std::sort(bm25_sorted.begin(), bm25_sorted.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - std::unordered_map emb_rank_map, bm25_rank_map; - for (size_t r = 0; r < emb_sorted.size(); ++r) { - emb_rank_map[emb_sorted[r].second] = r + 1; - } - for (size_t r = 0; r < bm25_sorted.size(); ++r) { - bm25_rank_map[bm25_sorted[r].second] = r + 1; - } - - std::vector> rrf_scored; - rrf_scored.reserve(num_items); - for (size_t i = 0; i < num_items; ++i) { - float rrf = RRF_EMB_WEIGHT / (RRF_K + emb_rank_map[i]) + - RRF_BM25_WEIGHT / (RRF_K + bm25_rank_map[i]); - rrf_scored.emplace_back(rrf, i); - } - - std::sort(rrf_scored.begin(), rrf_scored.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - return rrf_scored; -} - -static std::vector tokenize_words(const std::string& text) { - std::vector words; - std::string current; - for (char c : text) { - if (std::isalnum(static_cast(c))) { - current += std::tolower(static_cast(c)); - } else if (!current.empty()) { - if (current.length() > 2) { - words.push_back(current); - } - current.clear(); - } - } - if (!current.empty() && current.length() > 2) { - words.push_back(current); - } - return words; -} - -static float compute_bm25_score( - const std::vector& query_words, - const std::string& doc_content, - float avg_doc_len, - const std::unordered_map& doc_freqs, - int total_docs -) { - auto doc_words = tokenize_words(doc_content); - float doc_len = static_cast(doc_words.size()); - - std::unordered_map tf; - for (const auto& w : doc_words) { - tf[w]++; - } - - float score = 0.0f; - std::unordered_set query_set(query_words.begin(), query_words.end()); - - for (const auto& term : query_set) { - auto tf_it = tf.find(term); - if (tf_it == tf.end()) continue; - - float term_freq = static_cast(tf_it->second); - float df = 1.0f; - auto df_it = doc_freqs.find(term); - if (df_it != doc_freqs.end()) { - df = static_cast(df_it->second); - } - - float idf = std::log((total_docs - df + 0.5f) / (df + 0.5f) + 1.0f); - - float tf_component = (term_freq * (BM25_K1 + 1.0f)) / - (term_freq + BM25_K1 * (1.0f - BM25_B + BM25_B * (doc_len / avg_doc_len))); - - score += idf * tf_component; - } - - return score; -} - -std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query) { - if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { - return ""; - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) return ""; - - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) return ""; - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.size() != handle->corpus_embedding_dim) { - CACTUS_LOG_WARN("rag", "Query embedding dimension mismatch"); - return ""; - } - - index::QueryOptions options; - options.top_k = RAG_CANDIDATE_K; - options.score_threshold = 0.0f; - - std::vector> query_embeddings = {query_embedding}; - - try { - auto results = handle->corpus_index->query(query_embeddings, options); - if (results.empty() || results[0].empty()) { - return ""; - } - - std::vector doc_ids; - std::vector embedding_scores; - for (const auto& result : results[0]) { - doc_ids.push_back(result.doc_id); - embedding_scores.push_back(result.score); - } - - auto docs = handle->corpus_index->get_documents(doc_ids); - if (docs.empty()) { - return ""; - } - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& doc : docs) { - auto words = tokenize_words(doc.content); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = total_len / docs.size(); - - std::vector> emb_ranked; - std::vector> bm25_ranked; - for (size_t i = 0; i < docs.size(); ++i) { - emb_ranked.emplace_back(embedding_scores[i], i); - float bm25 = compute_bm25_score( - query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); - - std::string context = "[Retrieved Context - Use ONLY this information to answer. If the answer is not in the context, say \"I don't have enough information to answer that.\"]\n"; - size_t count = std::min(RAG_TOP_K, rrf_scored.size()); - for (size_t i = 0; i < count; ++i) { - size_t idx = rrf_scored[i].second; - context += "---\n"; - context += docs[idx].content; - if (!docs[idx].metadata.empty()) { - context += "\n(Source: " + docs[idx].metadata + ")"; - } - context += "\n"; - } - context += "---\n\n"; - - CACTUS_LOG_DEBUG("rag", "Retrieved " << count << " RAG chunks (hybrid BM25+embedding)"); - return context; - - } catch (const std::exception& e) { - CACTUS_LOG_WARN("rag", "RAG retrieval failed: " << e.what()); - return ""; - } -} - -static std::string tool_to_text(const ToolFunction& tool) { - return tool.name + " " + tool.description; -} - -static float cosine_similarity(const std::vector& a, const std::vector& b) { - if (a.size() != b.size() || a.empty()) return 0.0f; - - float dot = 0.0f, norm_a = 0.0f, norm_b = 0.0f; - for (size_t i = 0; i < a.size(); ++i) { - dot += a[i] * b[i]; - norm_a += a[i] * a[i]; - norm_b += b[i] * b[i]; - } - - float denom = std::sqrt(norm_a) * std::sqrt(norm_b); - return denom > 0.0f ? dot / denom : 0.0f; -} - -std::vector select_relevant_tools( - CactusModelHandle* handle, - const std::string& query, - const std::vector& all_tools, - size_t top_k -) { - using cactus::ffi::ToolFunction; - if (!handle || all_tools.empty() || top_k == 0) return all_tools; - - // If we have fewer tools than top_k, just return all - if (all_tools.size() <= top_k) return all_tools; - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_WARN("tool_rag", "No tokenizer available, returning all tools"); - return all_tools; - } - - // Build tool texts and embeddings if not cached or tool set changed - bool need_recompute = (handle->tool_texts.size() != all_tools.size()); - if (!need_recompute) { - for (size_t i = 0; i < all_tools.size(); ++i) { - if (tool_to_text(all_tools[i]) != handle->tool_texts[i]) { - need_recompute = true; - break; - } - } - } - - if (need_recompute) { - CACTUS_LOG_DEBUG("tool_rag", "Computing embeddings for " << all_tools.size() << " tools"); - handle->tool_texts.clear(); - handle->tool_embeddings.clear(); - handle->tool_texts.reserve(all_tools.size()); - handle->tool_embeddings.reserve(all_tools.size()); - - for (const auto& tool : all_tools) { - std::string text = tool_to_text(tool); - handle->tool_texts.push_back(text); - - std::vector tokens = tokenizer->encode(text); - if (!tokens.empty()) { - std::vector emb = handle->model->get_embeddings(tokens, true, true); - handle->tool_embeddings.push_back(std::move(emb)); - } else { - handle->tool_embeddings.push_back({}); - } - } - } - - // Embed the query - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) { - CACTUS_LOG_WARN("tool_rag", "Empty query, returning all tools"); - return all_tools; - } - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.empty()) { - CACTUS_LOG_WARN("tool_rag", "Failed to get query embedding, returning all tools"); - return all_tools; - } - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& text : handle->tool_texts) { - auto words = tokenize_words(text); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = total_len / handle->tool_texts.size(); - - std::vector> emb_ranked; - for (size_t i = 0; i < handle->tool_embeddings.size(); ++i) { - float sim = cosine_similarity(query_embedding, handle->tool_embeddings[i]); - emb_ranked.emplace_back(sim, i); - } - - std::vector> bm25_ranked; - for (size_t i = 0; i < handle->tool_texts.size(); ++i) { - float bm25 = compute_bm25_score( - query_words, handle->tool_texts[i], avg_doc_len, doc_freqs, handle->tool_texts.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, all_tools.size()); - - // Select top-k tools - std::vector selected; - selected.reserve(top_k); - for (size_t i = 0; i < top_k && i < rrf_scored.size(); ++i) { - size_t idx = rrf_scored[i].second; - selected.push_back(all_tools[idx]); - CACTUS_LOG_DEBUG("tool_rag", "Selected tool: " << all_tools[idx].name << " (RRF: " << rrf_scored[i].first << ")"); - } - - CACTUS_LOG_INFO("tool_rag", "Selected " << selected.size() << " of " << all_tools.size() << " tools using hybrid ranking"); - return selected; -} - -extern "C" { - -int cactus_rag_query( - cactus_model_t model, - const char* query, - char* response_buffer, - size_t buffer_size, - size_t top_k -) { - if (!model || !query || !response_buffer || buffer_size == 0) { - return -1; - } - - auto* handle = static_cast(model); - - if (!handle->corpus_index || handle->corpus_embedding_dim == 0) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No corpus index loaded\"}"); - return 0; - } - - try { - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"No tokenizer\"}"); - return 0; - } - - std::vector query_tokens = tokenizer->encode(query); - if (query_tokens.empty()) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Empty query\"}"); - return 0; - } - - std::vector query_embedding = handle->model->get_embeddings(query_tokens, true, true); - if (query_embedding.size() != handle->corpus_embedding_dim) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Embedding dimension mismatch\"}"); - return 0; - } - - index::QueryOptions options; - options.top_k = RAG_CANDIDATE_K; - options.score_threshold = 0.0f; - - std::vector> query_embeddings = {query_embedding}; - auto results = handle->corpus_index->query(query_embeddings, options); - - if (results.empty() || results[0].empty()) { - std::strcpy(response_buffer, "{\"chunks\":[]}"); - return 0; - } - - std::vector doc_ids; - std::vector embedding_scores; - for (const auto& result : results[0]) { - doc_ids.push_back(result.doc_id); - embedding_scores.push_back(result.score); - } - - auto docs = handle->corpus_index->get_documents(doc_ids); - - auto query_words = tokenize_words(query); - - float total_len = 0.0f; - std::unordered_map doc_freqs; - for (const auto& doc : docs) { - auto words = tokenize_words(doc.content); - total_len += words.size(); - std::unordered_set unique_words(words.begin(), words.end()); - for (const auto& w : unique_words) { - doc_freqs[w]++; - } - } - float avg_doc_len = docs.size() > 0 ? total_len / docs.size() : 1.0f; - - std::vector> emb_ranked; - std::vector> bm25_ranked; - for (size_t i = 0; i < docs.size(); ++i) { - emb_ranked.emplace_back(embedding_scores[i], i); - float bm25 = compute_bm25_score( - query_words, docs[i].content, avg_doc_len, doc_freqs, docs.size() - ); - bm25_ranked.emplace_back(bm25, i); - } - - auto rrf_scored = compute_rrf_scores(emb_ranked, bm25_ranked, docs.size()); - - size_t result_count = std::min(top_k > 0 ? top_k : RAG_TOP_K, rrf_scored.size()); - - std::ostringstream oss; - oss << "{\"chunks\":["; - for (size_t i = 0; i < result_count; ++i) { - size_t idx = rrf_scored[i].second; - float final_score = rrf_scored[i].first; - - if (i > 0) oss << ","; - oss << "{\"score\":" << std::setprecision(4) << final_score - << ",\"source\":\"" << docs[idx].metadata << "\"" - << ",\"content\":\""; - for (char c : docs[idx].content) { - switch (c) { - case '"': oss << "\\\""; break; - case '\\': oss << "\\\\"; break; - case '\n': oss << "\\n"; break; - case '\r': oss << "\\r"; break; - case '\t': oss << "\\t"; break; - default: oss << c; - } - } - oss << "\"}"; - } - oss << "]}"; - - std::string result = oss.str(); - if (result.size() >= buffer_size) { - std::strcpy(response_buffer, "{\"chunks\":[],\"error\":\"Buffer too small\"}"); - return -1; - } - - std::strcpy(response_buffer, result.c_str()); - return static_cast(result.size()); - - } catch (const std::exception& e) { - std::string error = "{\"chunks\":[],\"error\":\"" + std::string(e.what()) + "\"}"; - if (error.size() < buffer_size) { - std::strcpy(response_buffer, error.c_str()); - } - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_stream.cpp b/cactus/ffi/cactus_stream.cpp deleted file mode 100644 index 40dc6d6fa..000000000 --- a/cactus/ffi/cactus_stream.cpp +++ /dev/null @@ -1,2704 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_cloud.h" -#include "cactus_utils.h" -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::ffi; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::trim_mel_frames; - -double json_number(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return 0.0; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - size_t end = start; - while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; - try { return std::stod(json.substr(start, end - start)); } - catch (...) { return 0.0; } -} - -std::string json_string(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return {}; - size_t start = pos + pattern.size(); - - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start >= json.size() || json[start] != '"') return {}; - size_t q1 = start; - size_t q2 = json.find('"', q1 + 1); - if (q2 == std::string::npos) return {}; - return json.substr(q1 + 1, q2 - q1 - 1); -} - -inline std::string escape_json(const std::string& s) { - return escape_json_string(s); -} - -bool json_bool(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return false; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start + 4 <= json.size() && json.substr(start, 4) == "true") return true; - return false; -} - -static std::string ensure_json_bool_option(const std::string& json, - const std::string& key, - bool value) { - const std::string field = "\"" + key + "\""; - if (json.find(field) != std::string::npos) return json; - - const std::string bool_str = value ? "true" : "false"; - if (json.empty()) { - return "{\"" + key + "\":" + bool_str + "}"; - } - - std::string out = json; - const size_t close = out.rfind('}'); - const size_t open = out.find('{'); - if (open == std::string::npos || close == std::string::npos || close <= open) { - return json; - } - - size_t content_start = open + 1; - while (content_start < close && std::isspace(static_cast(out[content_start]))) { - ++content_start; - } - const bool has_entries = content_start < close; - const std::string insertion = (has_entries ? "," : "") + field + ":" + bool_str; - out.insert(close, insertion); - return out; -} - -std::string json_array(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return "[]"; - size_t start = pos + pattern.size(); - while (start < json.size() && (json[start] == ' ' || json[start] == '\t')) ++start; - if (start >= json.size() || json[start] != '[') return "[]"; - int depth = 1; - size_t end = start + 1; - while (end < json.size() && depth > 0) { - if (json[end] == '[') depth++; - else if (json[end] == ']') depth--; - end++; - } - return json.substr(start, end - start); -} - -static std::string suppress_unwanted_text(const std::string& text) { - static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); - std::string result = std::regex_replace(text, pattern, ""); - - size_t start = result.find_first_not_of(" \t\n\r"); - if (start == std::string::npos) return ""; - size_t end = result.find_last_not_of(" \t\n\r"); - return result.substr(start, end - start + 1); -} - -static bool is_pcm_chunk_silent(const uint8_t* pcm_buffer, size_t pcm_buffer_size, float rms_threshold) { - if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t)) return true; - - const size_t sample_count = pcm_buffer_size / sizeof(int16_t); - const auto* samples = reinterpret_cast(pcm_buffer); - double sum_sq = 0.0; - for (size_t i = 0; i < sample_count; ++i) { - const double normalized = static_cast(samples[i]) / 32768.0; - sum_sq += normalized * normalized; - } - - const double rms = std::sqrt(sum_sq / static_cast(sample_count)); - return rms < static_cast(rms_threshold); -} - -static void parse_stream_transcribe_init_options(const std::string& json, - double& confirmation_threshold, - size_t& min_chunk_size, - bool& telemetry_enabled, - std::string& language) { - confirmation_threshold = 0.99; - min_chunk_size = 32000; - telemetry_enabled = true; - language = "en"; - - if (json.empty()) return; - - size_t pos = json.find("\"confirmation_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - confirmation_threshold = std::stod(json.substr(pos)); - } - - pos = json.find("\"min_chunk_size\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - min_chunk_size = static_cast(std::stod(json.substr(pos))); - } - - pos = json.find("\"telemetry_enabled\""); - if (pos != std::string::npos) { - telemetry_enabled = json_bool(json, "telemetry_enabled"); - } - - language = json_string(json, "language"); - if (language.empty()) language = "en"; -} - -static bool build_parakeet_features_from_pcm( - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - const cactus::engine::Model* model, - std::vector& out_features) { - if (!pcm_buffer || pcm_buffer_size < sizeof(int16_t) || !model) { - return false; - } - - std::vector waveform = cactus::audio::pcm_buffer_to_float_samples( - pcm_buffer, pcm_buffer_size); - if (waveform.empty()) { - return false; - } - - const auto cfg = get_parakeet_spectrogram_config(); - const size_t mel_bins = std::max( - 1, static_cast(model->get_config().num_mel_bins)); - cactus::engine::AudioProcessor processor; - processor.init_mel_filters( - cfg.n_fft / 2 + 1, - mel_bins, - 0.0f, - 8000.0f, - WHISPER_SAMPLE_RATE); - - const size_t waveform_samples = waveform.size(); - apply_preemphasis(waveform, 0.97f); - out_features = processor.compute_spectrogram(waveform, cfg); - normalize_parakeet_log_mel(out_features, mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(out_features, mel_bins, valid_frames); - return !out_features.empty(); -} - -static size_t parakeet_tdt_pcm_bytes_to_encoder_frames( - const cactus::engine::Model* model, - size_t pcm_bytes) { - if (!model) return 0; - const auto cfg = get_parakeet_spectrogram_config(); - const double sample_count = static_cast(pcm_bytes) / 2.0; - const double feature_frames = sample_count / static_cast(cfg.hop_length); - const double subsampling = std::max(1, model->get_config().subsampling_factor); - return static_cast(std::llround(feature_frames / subsampling)); -} - -struct CactusStreamTranscribeHandle { - CactusModelHandle* model_handle; - - struct CactusStreamTranscribeOptions { - double confirmation_threshold; - size_t min_chunk_size; - std::string language; - } options; - std::string transcribe_options_json; - bool has_custom_vocabulary_bias = false; - std::vector custom_vocabulary; - - std::vector audio_buffer; - std::string previous_parakeet_pending; - size_t previous_parakeet_audio_buffer_size = 0; - size_t previous_parakeet_pending_ticks = 0; - size_t parakeet_silence_run_bytes = 0; - size_t parakeet_speech_run_bytes = 0; - bool parakeet_resume_guard_active = true; - std::string parakeet_committed_text; - size_t parakeet_chunk_cursor_bytes = 0; - float parakeet_committed_until_sec = 0.0f; - bool parakeet_onset_active = false; - size_t parakeet_onset_start_bytes = 0; - bool parakeet_tdt_chunked_stream = false; - struct ParakeetTdtDecodeContext { - std::vector audio_samples; - size_t samples_decoded_up_to = 0; - float audio_time_offset_sec = 0.0f; - cactus::engine::ParakeetTDTModel::ChunkStreamState decoder_state; - cactus::engine::AudioProcessor audio_processor; - size_t mel_bins = 0; - bool initialized = false; - } parakeet_tdt_decode_context; - bool parakeet_tdt_chunked_blocked_until_pause = false; - cactus::engine::ParakeetTDTModel::ChunkStreamState parakeet_tdt_decoder_state; - std::string parakeet_tdt_pending_raw; - struct ParakeetTdtResumeCheckpoint { - size_t end_bytes = 0; - cactus::engine::ParakeetTDTModel::ChunkStreamState state; - }; - std::deque parakeet_tdt_resume_checkpoints; - - std::vector previous_segments; - bool previous_cloud_handoff = false; - float stream_audio_offset_sec = 0.0f; - uint64_t next_cloud_job_id = 1; - - struct CloudJob { - uint64_t id; - std::future result; - }; - std::vector pending_cloud_jobs; - std::vector> completed_cloud_results; - - char transcribe_response_buffer[65536]; - - std::chrono::steady_clock::time_point stream_start; - bool stream_first_token_seen; - double stream_first_token_ms; - int stream_total_tokens; - - std::chrono::steady_clock::time_point stream_session_start; - bool stream_session_first_token_seen; - double stream_session_first_token_ms; - int stream_cumulative_tokens; - double last_stream_decode_tps = 0.0; - double last_stream_decode_tokens = 0.0; - double last_stream_total_time_ms = 0.0; - double last_stream_raw_decoder_tps = 0.0; -}; - -static std::vector parse_segments(const std::string& transcribe_json); - -struct StreamWindowDecodeResult { - std::string raw_json; - std::string response; - std::vector segments; - double decode_tokens = 0.0; - bool cloud_handoff = false; -}; - -static double pcm_bytes_to_seconds(size_t pcm_bytes) { - return static_cast(pcm_bytes) / (16000.0 * 2.0); -} - -static size_t seconds_to_pcm_bytes(double seconds) { - if (seconds <= 0.0) return 0; - const size_t raw = static_cast(std::llround(seconds * 16000.0 * 2.0)); - return raw - (raw % sizeof(int16_t)); -} - -static std::string join_segments_by_end_time( - const std::vector& segments, - float offset_sec, - float min_end_exclusive_sec, - float max_end_inclusive_sec, - float* out_max_emitted_end_sec = nullptr) { - std::string joined; - float max_end_sec = min_end_exclusive_sec; - for (const auto& seg : segments) { - const float abs_end_sec = offset_sec + seg.end; - if (abs_end_sec <= min_end_exclusive_sec || abs_end_sec > max_end_inclusive_sec) { - continue; - } - if (!joined.empty()) joined += ' '; - joined += seg.text; - if (abs_end_sec > max_end_sec) { - max_end_sec = abs_end_sec; - } - } - - if (out_max_emitted_end_sec) { - *out_max_emitted_end_sec = max_end_sec; - } - return suppress_unwanted_text(joined); -} - -static bool run_stream_window_transcribe( - CactusStreamTranscribeHandle* handle, - const char* transcribe_prompt, - const std::string& effective_transcribe_options_json, - const uint8_t* decode_pcm, - size_t decode_pcm_size, - StreamWindowDecodeResult& out) { - if (!handle || !handle->model_handle || !decode_pcm || decode_pcm_size == 0) { - return false; - } - - const auto model_type = handle->model_handle->model->get_config().model_type; - const bool use_model_stream_mode = - model_type != cactus::engine::Config::ModelType::PARAKEET_TDT; - cactus::telemetry::setStreamMode(use_model_stream_mode); - const int result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - decode_pcm, - decode_pcm_size); - cactus::telemetry::setStreamMode(false); - - if (result < 0) { - return false; - } - - out.raw_json.assign(handle->transcribe_response_buffer); - out.response = suppress_unwanted_text(json_string(out.raw_json, "response")); - out.segments = parse_segments(out.raw_json); - out.decode_tokens = std::max(0.0, json_number(out.raw_json, "decode_tokens")); - out.cloud_handoff = json_bool(out.raw_json, "cloud_handoff"); - handle->last_stream_decode_tps = std::max(0.0, json_number(out.raw_json, "decode_tps")); - handle->last_stream_decode_tokens = out.decode_tokens; - handle->last_stream_total_time_ms = std::max(0.0, json_number(out.raw_json, "total_time_ms")); - handle->last_stream_raw_decoder_tps = std::max(0.0, json_number(out.raw_json, "raw_decoder_tps")); - return true; -} - -static bool run_parakeet_tdt_chunk_decode( - CactusStreamTranscribeHandle* handle, - size_t window_start_bytes, - size_t window_end_bytes, - size_t chunk_start_bytes, - size_t chunk_end_bytes, - cactus::engine::ParakeetTDTModel::ChunkStreamResult& out, - cactus::engine::ParakeetTDTModel::ChunkStreamState& out_state) { - if (!handle || !handle->model_handle || !handle->model_handle->model) { - return false; - } - if (chunk_end_bytes <= chunk_start_bytes || - window_end_bytes <= window_start_bytes || - window_end_bytes > handle->audio_buffer.size() || - chunk_end_bytes > window_end_bytes || - window_start_bytes > chunk_start_bytes) { - return false; - } - - auto* model = static_cast( - handle->model_handle->model.get()); - size_t replay_start_bytes = window_start_bytes; - cactus::engine::ParakeetTDTModel::ChunkStreamState base_state; - bool found_checkpoint = false; - for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); - it != handle->parakeet_tdt_resume_checkpoints.rend(); - ++it) { - if (it->end_bytes <= window_start_bytes) { - replay_start_bytes = std::max(window_start_bytes, it->end_bytes); - base_state = it->state; - found_checkpoint = true; - break; - } - } - if (!found_checkpoint) { - for (auto it = handle->parakeet_tdt_resume_checkpoints.rbegin(); - it != handle->parakeet_tdt_resume_checkpoints.rend(); - ++it) { - if (it->end_bytes <= chunk_start_bytes) { - replay_start_bytes = std::max(window_start_bytes, it->end_bytes); - base_state = it->state; - found_checkpoint = true; - break; - } - } - } - if (!found_checkpoint) { - replay_start_bytes = window_start_bytes; - base_state = {}; - } - - std::vector audio_features; - if (!build_parakeet_features_from_pcm( - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - model, - audio_features)) { - return false; - } - - const size_t replay_start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, replay_start_bytes - window_start_bytes); - const size_t start_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, chunk_start_bytes - window_start_bytes); - size_t end_frame = parakeet_tdt_pcm_bytes_to_encoder_frames( - model, chunk_end_bytes - window_start_bytes); - if (end_frame <= start_frame) { - end_frame = start_frame + 1; - } - - const auto decode_start = std::chrono::steady_clock::now(); - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - out_state = base_state; - out = model->decode_chunk_stream( - audio_features, - replay_start_frame, - start_frame, - end_frame, - out_state); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - const auto decode_end = std::chrono::steady_clock::now(); - const double decode_ms = std::chrono::duration_cast( - decode_end - decode_start).count() / 1000.0; - handle->last_stream_total_time_ms = decode_ms; - handle->last_stream_decode_tokens = static_cast(out.token_count); - handle->last_stream_decode_tps = - (out.token_count > 1 && decode_ms > 0.0) - ? ((static_cast(out.token_count) - 1.0) * 1000.0) / decode_ms - : 0.0; - handle->last_stream_raw_decoder_tps = out.raw_decoder_tps; - return true; -} - -static void reset_parakeet_tdt_chunk_stream(CactusStreamTranscribeHandle* handle) { - if (!handle) return; - handle->parakeet_tdt_decoder_state = {}; - handle->parakeet_tdt_pending_raw.clear(); - handle->parakeet_tdt_resume_checkpoints.clear(); -} - -static std::vector> collect_parakeet_word_spans(const std::string& text); - -static std::string strip_unwanted_text_preserving_edges(const std::string& text) { - static const std::regex pattern(R"(\([^)]*\)|\[[^\]]*\]|\.\.\.)"); - return std::regex_replace(text, pattern, ""); -} - -static void parakeet_append_monotonic_text(std::string& dst, const std::string& piece) { - const std::string normalized = suppress_unwanted_text(piece); - if (normalized.empty()) return; - if (dst.empty()) { - dst = normalized; - return; - } - - const unsigned char first = static_cast(normalized.front()); - const bool attach_without_space = - std::ispunct(first) && normalized.front() != '"' && - normalized.front() != '\'' && normalized.front() != '('; - if (!std::isspace(static_cast(dst.back())) && !attach_without_space) { - dst += " "; - } - dst += normalized; -} - -static std::vector parse_segments(const std::string& transcribe_json) { - std::vector segs; - std::string arr = json_array(transcribe_json, "segments"); - size_t i = 1; - while (i < arr.size() - 1) { - while (i < arr.size() && (arr[i] == ',' || std::isspace((unsigned char)arr[i]))) ++i; - if (i >= arr.size() - 1 || arr[i] != '{') break; - int depth = 1; - size_t obj_start = i++; - while (i < arr.size() && depth > 0) { - if (arr[i] == '{') ++depth; - else if (arr[i] == '}') --depth; - ++i; - } - std::string obj = arr.substr(obj_start, i - obj_start); - segs.emplace_back( - static_cast(json_number(obj, "start")), - static_cast(json_number(obj, "end")), - json_string(obj, "text") - ); - } - return segs; -} - -static bool is_parakeet_word_char(unsigned char ch) { - return std::isalnum(ch) || ch == '\''; -} - -static std::vector> collect_parakeet_word_spans(const std::string& text) { - std::vector> spans; - bool in_word = false; - size_t word_start = 0; - - for (size_t i = 0; i < text.size(); ++i) { - const unsigned char ch = static_cast(text[i]); - if (is_parakeet_word_char(ch)) { - if (!in_word) { - in_word = true; - word_start = i; - } - } else if (in_word) { - spans.emplace_back(word_start, i); - in_word = false; - } - } - - if (in_word) { - spans.emplace_back(word_start, text.size()); - } - return spans; -} - -static std::string parakeet_word_from_span( - const std::string& text, - const std::pair& span) { - if (span.second <= span.first || span.second > text.size()) return ""; - return text.substr(span.first, span.second - span.first); -} - -static bool parakeet_words_fuzzy_equal(const std::string& a, const std::string& b) { - if (a.empty() || b.empty()) return false; - const size_t dist = levenshtein_ci(a, b); - if (dist == 0) return true; - const size_t max_len = std::max(a.size(), b.size()); - return max_len >= 5 && dist <= 1; -} - -static size_t parakeet_common_prefix_words(const std::string& left, const std::string& right) { - const auto left_spans = collect_parakeet_word_spans(left); - const auto right_spans = collect_parakeet_word_spans(right); - const size_t max_words = std::min(left_spans.size(), right_spans.size()); - size_t common = 0; - for (; common < max_words; ++common) { - const std::string left_word = parakeet_word_from_span(left, left_spans[common]); - const std::string right_word = parakeet_word_from_span(right, right_spans[common]); - if (!parakeet_words_fuzzy_equal(left_word, right_word)) { - break; - } - } - return common; -} - -static std::string parakeet_take_first_words(const std::string& text, size_t words) { - if (text.empty() || words == 0) return ""; - const auto spans = collect_parakeet_word_spans(text); - if (spans.empty()) return ""; - if (words >= spans.size()) return suppress_unwanted_text(text); - const size_t end_idx = spans[words].first; - return suppress_unwanted_text(text.substr(0, end_idx)); -} - -static std::string parakeet_drop_first_words(const std::string& text, size_t words) { - if (text.empty()) return ""; - const auto spans = collect_parakeet_word_spans(text); - if (spans.empty()) return suppress_unwanted_text(text); - if (words >= spans.size()) return ""; - const size_t start_idx = spans[words].first; - return suppress_unwanted_text(text.substr(start_idx)); -} - -static size_t parakeet_word_count(const std::string& text) { - return collect_parakeet_word_spans(text).size(); -} - -static size_t parakeet_overlap_suffix_prefix_words( - const std::string& left, - const std::string& right, - size_t max_words = 64) { - const auto left_spans = collect_parakeet_word_spans(left); - const auto right_spans = collect_parakeet_word_spans(right); - if (left_spans.empty() || right_spans.empty()) return 0; - - const size_t max_overlap = std::min({left_spans.size(), right_spans.size(), max_words}); - size_t best = 0; - for (size_t k = 1; k <= max_overlap; ++k) { - bool matched = true; - for (size_t i = 0; i < k; ++i) { - const auto& left_span = left_spans[left_spans.size() - k + i]; - const auto& right_span = right_spans[i]; - const std::string left_word = parakeet_word_from_span(left, left_span); - const std::string right_word = parakeet_word_from_span(right, right_span); - if (!parakeet_words_fuzzy_equal(left_word, right_word)) { - matched = false; - break; - } - } - if (matched) best = k; - } - return best; -} - -static std::string parakeet_strip_recent_aligned_overlap( - const std::string& committed_text, - const std::string& hypothesis, - size_t max_recent_words = 24) { - const std::string candidate = suppress_unwanted_text(hypothesis); - if (candidate.empty() || committed_text.empty()) return candidate; - - const auto committed_spans = collect_parakeet_word_spans(committed_text); - if (committed_spans.empty()) return candidate; - - const size_t recent_words = std::min(committed_spans.size(), max_recent_words); - const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; - const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); - const auto recent_spans = collect_parakeet_word_spans(recent_tail); - const auto candidate_spans = collect_parakeet_word_spans(candidate); - const size_t recent_count = recent_spans.size(); - const size_t candidate_count = candidate_spans.size(); - if (recent_count == 0 || candidate_count == 0) return candidate; - - size_t best_run = 0; - size_t best_candidate_start = candidate_count; - size_t best_candidate_end = 0; - size_t best_recent_start = recent_count; - - for (size_t i = 0; i < recent_count; ++i) { - for (size_t j = 0; j < candidate_count; ++j) { - size_t run = 0; - while (i + run < recent_count && j + run < candidate_count) { - const std::string recent_word = parakeet_word_from_span( - recent_tail, recent_spans[i + run]); - const std::string candidate_word = parakeet_word_from_span( - candidate, candidate_spans[j + run]); - if (!parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - break; - } - ++run; - } - - const size_t candidate_end = j + run; - const bool better_run = run > best_run; - const bool better_end = run == best_run && candidate_end > best_candidate_end; - const bool better_start = run == best_run && - candidate_end == best_candidate_end && - j < best_candidate_start; - if (better_run || better_end || better_start) { - best_run = run; - best_candidate_start = j; - best_candidate_end = candidate_end; - best_recent_start = i; - } - } - } - - if (best_run == 0 || best_candidate_end == 0) return candidate; - - const size_t min_overlap_words = candidate_count <= 5 ? 2 : 3; - if (best_run < min_overlap_words) return candidate; - - if (best_candidate_start > 1) return candidate; - if (best_candidate_start == 1) { - const std::string lead_word = parakeet_word_from_span(candidate, candidate_spans[0]); - const std::string overlap_word = parakeet_word_from_span(candidate, candidate_spans[1]); - const std::string recent_word = parakeet_word_from_span( - recent_tail, recent_spans[best_recent_start]); - if (!parakeet_words_fuzzy_equal(lead_word, overlap_word) && - !parakeet_words_fuzzy_equal(lead_word, recent_word)) { - return candidate; - } - } - - return parakeet_drop_first_words(candidate, best_candidate_end); -} - -static std::string parakeet_strip_recent_committed_prefix( - const std::string& committed_text, - const std::string& hypothesis); - -static std::string parakeet_emit_delta( - const std::string& committed_text, - const std::string& newly_confirmed) { - std::string candidate = parakeet_strip_recent_committed_prefix( - committed_text, newly_confirmed); - if (candidate.empty()) return ""; - if (committed_text.empty()) return candidate; - - const size_t overlap_words = parakeet_overlap_suffix_prefix_words(committed_text, candidate); - if (overlap_words > 0) { - return parakeet_drop_first_words(candidate, overlap_words); - } - - return candidate; -} - -static std::string parakeet_strip_recent_committed_prefix( - const std::string& committed_text, - const std::string& hypothesis) { - const std::string candidate = suppress_unwanted_text(hypothesis); - if (candidate.empty() || committed_text.empty()) return candidate; - - const auto committed_spans = collect_parakeet_word_spans(committed_text); - if (committed_spans.empty()) return candidate; - - const size_t recent_words = std::min(committed_spans.size(), 24); - const size_t recent_start = committed_spans[committed_spans.size() - recent_words].first; - const std::string recent_tail = suppress_unwanted_text(committed_text.substr(recent_start)); - const size_t candidate_words = parakeet_word_count(candidate); - const size_t overlap_words = parakeet_overlap_suffix_prefix_words(recent_tail, candidate, recent_words); - - // Be conservative for tiny hypotheses, but strip even a single repeated - // boundary word when the new hypothesis is a longer continuation. - const size_t required_overlap = candidate_words >= 5 ? 1 : 2; - if (overlap_words >= required_overlap) { - return parakeet_drop_first_words(candidate, overlap_words); - } - - const std::string aligned_trimmed = parakeet_strip_recent_aligned_overlap( - committed_text, candidate, recent_words); - if (aligned_trimmed != candidate) { - return aligned_trimmed; - } - - const auto recent_spans = collect_parakeet_word_spans(recent_tail); - const auto candidate_spans = collect_parakeet_word_spans(candidate); - const size_t recent_count = recent_spans.size(); - const size_t candidate_count = candidate_spans.size(); - if (recent_count == 0 || candidate_count < 4) return candidate; - - std::vector> dp( - recent_count + 1, std::vector(candidate_count + 1, 0)); - for (size_t i = 1; i <= recent_count; ++i) { - for (size_t j = 1; j <= candidate_count; ++j) { - const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); - const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); - if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - dp[i][j] = dp[i - 1][j - 1] + 1; - } else { - dp[i][j] = std::max(dp[i - 1][j], dp[i][j - 1]); - } - } - } - - const size_t lcs_words = dp[recent_count][candidate_count]; - if (lcs_words < 3 || lcs_words * 2 < candidate_count) return candidate; - - size_t i = recent_count; - size_t j = candidate_count; - size_t last_matched_candidate_word = 0; - bool found_match = false; - bool captured_last_match = false; - while (i > 0 && j > 0) { - const std::string recent_word = parakeet_word_from_span(recent_tail, recent_spans[i - 1]); - const std::string candidate_word = parakeet_word_from_span(candidate, candidate_spans[j - 1]); - if (parakeet_words_fuzzy_equal(recent_word, candidate_word)) { - if (!captured_last_match) { - last_matched_candidate_word = j; - captured_last_match = true; - } - found_match = true; - --i; - --j; - } else if (dp[i - 1][j] >= dp[i][j - 1]) { - --i; - } else { - --j; - } - } - - if (!found_match || last_matched_candidate_word >= candidate_count) return candidate; - return parakeet_drop_first_words(candidate, last_matched_candidate_word); - - return candidate; -} - -static std::string serialize_segments_with_offset( - const std::vector& segs, float offset_sec) { - std::ostringstream out; - out << "["; - for (size_t i = 0; i < segs.size(); ++i) { - if (i > 0) out << ","; - out << std::fixed << std::setprecision(3) - << "{\"start\":" << (segs[i].start + offset_sec) - << ",\"end\":" << (segs[i].end + offset_sec) - << ",\"text\":\"" << escape_json(segs[i].text) << "\"}"; - } - out << "]"; - return out.str(); -} - -static std::string build_stream_response( - const std::string& raw_json_str, - const std::string& error_msg, - const std::string& confirmed, - const std::string& pending, - const std::string& segments_json, - bool cloud_handoff, - double buffer_duration_ms, - uint64_t cloud_job_id, - uint64_t cloud_result_job_id, - const CloudResponse& cloud_result, - double override_time_to_first_token_ms = -1.0, - double override_total_time_ms = -1.0, - double override_decode_tps = -1.0, - double override_decode_tokens = -1.0, - double override_total_tokens = -1.0, - double override_raw_decoder_tps = -1.0 -) { - std::string function_calls = json_array(raw_json_str, "function_calls"); - double confidence = json_number(raw_json_str, "confidence"); - double time_to_first_token_ms = json_number(raw_json_str, "time_to_first_token_ms"); - double total_time_ms = json_number(raw_json_str, "total_time_ms"); - double prefill_tps = json_number(raw_json_str, "prefill_tps"); - double decode_tps = json_number(raw_json_str, "decode_tps"); - double ram_usage_mb = json_number(raw_json_str, "ram_usage_mb"); - double prefill_tokens = json_number(raw_json_str, "prefill_tokens"); - double decode_tokens = json_number(raw_json_str, "decode_tokens"); - double total_tokens = json_number(raw_json_str, "total_tokens"); - double raw_decoder_tps = json_number(raw_json_str, "raw_decoder_tps"); - if (override_time_to_first_token_ms >= 0.0) { - time_to_first_token_ms = override_time_to_first_token_ms; - } - if (override_total_time_ms >= 0.0) { - total_time_ms = override_total_time_ms; - } - if (override_decode_tps >= 0.0) { - decode_tps = override_decode_tps; - } - if (override_decode_tokens >= 0.0) { - decode_tokens = override_decode_tokens; - } - if (override_total_tokens >= 0.0) { - total_tokens = override_total_tokens; - } - if (override_raw_decoder_tps >= 0.0) { - raw_decoder_tps = override_raw_decoder_tps; - } - std::string effective_confirmed = confirmed; - if (cloud_result.used_cloud && !cloud_result.transcript.empty()) { - effective_confirmed = cloud_result.transcript; - } - - std::ostringstream json_builder; - json_builder << "{"; - json_builder << "\"success\":true,"; - json_builder << "\"buffer_duration_ms\":" << buffer_duration_ms << ","; - json_builder << "\"error\":" << (error_msg.empty() ? "null" : "\"" + escape_json(error_msg) + "\"") << ","; - json_builder << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; - json_builder << "\"cloud_job_id\":" << cloud_job_id << ","; - json_builder << "\"cloud_result_job_id\":" << cloud_result_job_id << ","; - json_builder << "\"cloud_result\":\"" << escape_json(cloud_result.transcript) << "\","; - json_builder << "\"cloud_result_used_cloud\":" << (cloud_result.used_cloud ? "true" : "false") << ","; - json_builder << "\"cloud_result_error\":"; - if (cloud_result.error.empty()) { - json_builder << "null,"; - } else { - json_builder << "\"" << escape_json(cloud_result.error) << "\","; - } - json_builder << "\"cloud_result_source\":\"" << (cloud_result.used_cloud ? "cloud" : "fallback") << "\","; - json_builder << "\"confirmed_local\":\"" << escape_json(confirmed) << "\","; - json_builder << "\"confirmed\":\"" << escape_json(effective_confirmed) << "\","; - json_builder << "\"pending\":\"" << escape_json(pending) << "\","; - json_builder << "\"segments\":" << segments_json << ","; - json_builder << "\"function_calls\":" << function_calls << ","; - json_builder << "\"confidence\":" << confidence << ","; - json_builder << "\"time_to_first_token_ms\":" << time_to_first_token_ms << ","; - json_builder << "\"total_time_ms\":" << total_time_ms << ","; - json_builder << "\"prefill_tps\":" << prefill_tps << ","; - json_builder << "\"decode_tps\":" << decode_tps << ","; - json_builder << "\"ram_usage_mb\":" << ram_usage_mb << ","; - json_builder << "\"prefill_tokens\":" << prefill_tokens << ","; - json_builder << "\"decode_tokens\":" << decode_tokens << ","; - json_builder << "\"total_tokens\":" << total_tokens << ","; - json_builder << "\"raw_decoder_tps\":" << raw_decoder_tps; - json_builder << "}"; - return json_builder.str(); -} - -extern "C" { - -cactus_stream_transcribe_t cactus_stream_transcribe_start(cactus_model_t model, const char* options_json) { - if (!model) { - last_error_message = "Model not initialized. Check model path and files."; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } - - try { - auto* model_handle = static_cast(model); - if (!model_handle->model) { - last_error_message = "Invalid model handle."; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } - - auto* stream_handle = new CactusStreamTranscribeHandle(); - stream_handle->model_handle = model_handle; - stream_handle->transcribe_response_buffer[0] = '\0'; - - auto session_start_time = std::chrono::steady_clock::now(); - stream_handle->stream_start = session_start_time; - stream_handle->stream_first_token_seen = false; - stream_handle->stream_first_token_ms = 0.0; - stream_handle->stream_total_tokens = 0; - - stream_handle->stream_session_start = session_start_time; - stream_handle->stream_session_first_token_seen = false; - stream_handle->stream_session_first_token_ms = 0.0; - stream_handle->stream_cumulative_tokens = 0; - - double confirmation_threshold; - size_t min_chunk_size; - bool telemetry_enabled; - std::string language; - parse_stream_transcribe_init_options( - options_json ? options_json : "", - confirmation_threshold, - min_chunk_size, - telemetry_enabled, - language - ); - - stream_handle->options = { confirmation_threshold, min_chunk_size, language }; - stream_handle->transcribe_options_json = options_json ? options_json : ""; - stream_handle->parakeet_tdt_chunked_stream = - model_handle->model->get_config().model_type == - cactus::engine::Config::ModelType::PARAKEET_TDT; - { - float vocabulary_boost = 5.0f; - parse_custom_vocabulary_options(stream_handle->transcribe_options_json, - stream_handle->custom_vocabulary, vocabulary_boost); - auto vocab_bias = build_custom_vocabulary_bias( - model_handle->model->get_tokenizer(), - stream_handle->custom_vocabulary, - vocabulary_boost - ); - stream_handle->has_custom_vocabulary_bias = !vocab_bias.empty(); - if (stream_handle->has_custom_vocabulary_bias) { - model_handle->model->set_vocab_bias(vocab_bias); - } - } - - if (stream_handle->parakeet_tdt_chunked_stream) { - auto& ctx = stream_handle->parakeet_tdt_decode_context; - ctx.mel_bins = std::max( - 1, static_cast(model_handle->model->get_config().num_mel_bins)); - auto cfg = get_parakeet_spectrogram_config(); - ctx.audio_processor.init_mel_filters( - cfg.n_fft / 2 + 1, - ctx.mel_bins, - 0.0f, - 8000.0f, - WHISPER_SAMPLE_RATE); - ctx.initialized = true; - } - - CACTUS_LOG_INFO("stream_transcribe_start", - "Stream transcription initialized for model: " << model_handle->model_name); - - return stream_handle; - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_start: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } catch (...) { - last_error_message = "Unknown exception during stream transcription initialization"; - CACTUS_LOG_ERROR("stream_transcribe_start", last_error_message); - return nullptr; - } -} - -int cactus_stream_transcribe_process( - cactus_stream_transcribe_t stream, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size, - char* response_buffer, - size_t buffer_size -) { - if (!stream) { - last_error_message = "Stream not initialized."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - if (!pcm_buffer || pcm_buffer_size == 0) { - last_error_message = "Invalid parameters: pcm_buffer or pcm_buffer_size"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - if (!response_buffer || buffer_size == 0) { - last_error_message = "Invalid parameters: response_buffer or buffer_size"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - try { - auto* handle = static_cast(stream); - const auto model_type = handle->model_handle->model->get_config().model_type; - bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = - model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet_ctc = - model_type == cactus::engine::Config::ModelType::PARAKEET; - bool is_parakeet = - is_parakeet_ctc || is_parakeet_tdt; - bool is_gemma4 = model_type == cactus::engine::Config::ModelType::GEMMA4; - - if (is_parakeet_tdt && - handle->parakeet_tdt_chunked_stream && - handle->parakeet_tdt_decode_context.initialized) { - auto& ctx = handle->parakeet_tdt_decode_context; - auto* tdt_model = static_cast( - handle->model_handle->model.get()); - auto new_samples = cactus::audio::pcm_buffer_to_float_samples( - pcm_buffer, pcm_buffer_size); - ctx.audio_samples.insert(ctx.audio_samples.end(), new_samples.begin(), new_samples.end()); - - constexpr size_t kTdtLeftContextSamples = 8 * 16000; - constexpr size_t kTdtRightContextSamples = 16000; - constexpr size_t kTdtChunkSamples = 16000; - constexpr size_t kTdtColdStartSamples = 6 * 16000; - - const size_t total_samples = ctx.audio_samples.size(); - const size_t decodable_up_to = total_samples > kTdtRightContextSamples - ? total_samples - kTdtRightContextSamples - : 0; - const size_t min_chunk_samples = ctx.samples_decoded_up_to == 0 - ? kTdtColdStartSamples - : kTdtChunkSamples; - - if (decodable_up_to <= ctx.samples_decoded_up_to || - decodable_up_to - ctx.samples_decoded_up_to < min_chunk_samples) { - const std::string json_response = build_stream_response( - "{}", - "", - "", - handle->previous_parakeet_pending, - "[]", - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples - ? ctx.samples_decoded_up_to - kTdtLeftContextSamples - : 0; - const size_t window_end_sample = std::min( - total_samples, decodable_up_to + kTdtRightContextSamples); - std::vector window_audio( - ctx.audio_samples.begin() + window_start_sample, - ctx.audio_samples.begin() + window_end_sample); - - auto cfg = get_parakeet_spectrogram_config(); - const size_t waveform_samples = window_audio.size(); - apply_preemphasis(window_audio, 0.97f); - window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); - normalize_parakeet_log_mel(window_audio, ctx.mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); - - const uint32_t subsampling = - std::max(1, handle->model_handle->model->get_config().subsampling_factor); - const size_t samples_per_enc_frame = cfg.hop_length * subsampling; - const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; - const size_t decode_start_frame = context_samples / samples_per_enc_frame; - const size_t new_samples_count = decodable_up_to - ctx.samples_decoded_up_to; - const size_t decode_end_frame = - decode_start_frame + (new_samples_count / samples_per_enc_frame); - - if (decode_end_frame <= decode_start_frame) { - const std::string json_response = build_stream_response( - "{}", - "", - "", - handle->previous_parakeet_pending, - "[]", - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - { - const auto decode_start = std::chrono::steady_clock::now(); - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - chunk_decode = tdt_model->decode_chunk_stream( - window_audio, - decode_start_frame, - decode_start_frame, - decode_end_frame, - ctx.decoder_state); - cactus_reset(handle->model_handle); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - const auto decode_end = std::chrono::steady_clock::now(); - const double decode_ms = - std::chrono::duration_cast( - decode_end - decode_start).count() / 1000.0; - handle->last_stream_total_time_ms = decode_ms; - handle->last_stream_decode_tokens = static_cast(chunk_decode.token_count); - handle->last_stream_decode_tps = - (chunk_decode.token_count > 1 && decode_ms > 0.0) - ? ((static_cast(chunk_decode.token_count) - 1.0) * 1000.0) / decode_ms - : 0.0; - handle->last_stream_raw_decoder_tps = chunk_decode.raw_decoder_tps; - } - - const float window_offset_sec = ctx.audio_time_offset_sec + - static_cast(window_start_sample) / static_cast(WHISPER_SAMPLE_RATE); - std::vector confirmed_segments; - if (!chunk_decode.confirmed_text.empty()) { - confirmed_segments.emplace_back( - window_offset_sec + chunk_decode.start_sec, - window_offset_sec + chunk_decode.confirmed_end_sec, - suppress_unwanted_text(chunk_decode.confirmed_text)); - } - std::string confirmed_text = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_text = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_text.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_text, handle->custom_vocabulary); - } - if (!pending_text.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_text, handle->custom_vocabulary); - } - - if (!confirmed_text.empty()) { - if (chunk_decode.token_count > 0) { - handle->stream_total_tokens += static_cast(chunk_decode.token_count); - handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); - } - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - parakeet_append_monotonic_text(handle->parakeet_committed_text, confirmed_text); - } - - handle->previous_parakeet_pending = pending_text; - handle->previous_parakeet_pending_ticks = pending_text.empty() ? 0 : 1; - - if (chunk_decode.confirmed_token_count > 0) { - const size_t confirmed_samples_in_window = static_cast( - chunk_decode.confirmed_end_sec * WHISPER_SAMPLE_RATE); - ctx.samples_decoded_up_to = window_start_sample + confirmed_samples_in_window; - } else if (chunk_decode.token_count == 0) { - ctx.samples_decoded_up_to = decodable_up_to; - } - - if (ctx.samples_decoded_up_to > kTdtLeftContextSamples * 2) { - const size_t trim_samples = ctx.samples_decoded_up_to - kTdtLeftContextSamples; - ctx.audio_samples.erase( - ctx.audio_samples.begin(), - ctx.audio_samples.begin() + trim_samples); - ctx.samples_decoded_up_to -= trim_samples; - ctx.audio_time_offset_sec += static_cast(trim_samples) / 16000.0f; - } - - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - window_offset_sec + - (chunk_decode.confirmed_token_count > 0 ? chunk_decode.confirmed_end_sec : 0.0f)); - - const std::string json_response = build_stream_response( - "{}", - "", - confirmed_text, - pending_text, - serialize_segments_with_offset(confirmed_segments, 0.0f), - false, - 0.0, - 0, - 0, - CloudResponse{}, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - handle->last_stream_total_time_ms, - handle->last_stream_decode_tps, - handle->last_stream_decode_tokens, - handle->last_stream_decode_tokens, - handle->last_stream_raw_decoder_tps); - if (json_response.length() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - handle->audio_buffer.insert( - handle->audio_buffer.end(), - pcm_buffer, - pcm_buffer + pcm_buffer_size - ); - CACTUS_LOG_DEBUG("stream_transcribe_process", - "Inserted " << pcm_buffer_size << " bytes, buffer size: " << handle->audio_buffer.size()); - - if (handle->audio_buffer.size() < handle->options.min_chunk_size * sizeof(int16_t)) { - std::string json_response = "{\"success\":true,\"confirmed\":\"\",\"pending\":\"\"}"; - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; - const char* transcribe_prompt = - is_gemma4 ? "Transcribe the audio." : - (is_moonshine || is_parakeet) ? "" : - whisper_prompt.c_str(); - std::string effective_transcribe_options_json = handle->transcribe_options_json; - if (is_parakeet && !is_parakeet_tdt) { - effective_transcribe_options_json = ensure_json_bool_option( - effective_transcribe_options_json, "use_vad", false); - } - - if (is_parakeet_tdt) { - constexpr double kParakeetTdtLeftContextSec = 2.0; - constexpr double kParakeetTdtChunkSec = 0.25; - constexpr double kParakeetTdtRightContextSec = 1.0; - constexpr size_t kParakeetTdtMaxChunksPerProcess = 3; - - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - const size_t chunk_bytes = std::max( - seconds_to_pcm_bytes(kParakeetTdtChunkSec), - handle->options.min_chunk_size * sizeof(int16_t)); - const size_t right_context_bytes = seconds_to_pcm_bytes(kParakeetTdtRightContextSec); - const std::string tdt_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", true); - const std::string tdt_no_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", false); - - if (handle->parakeet_tdt_chunked_stream && - !handle->parakeet_tdt_chunked_blocked_until_pause && - !handle->parakeet_committed_text.empty()) { - const size_t encoder_frame_bytes = - get_parakeet_spectrogram_config().hop_length * - std::max(1, handle->model_handle->model->get_config().subsampling_factor) * - sizeof(int16_t); - std::string confirmed; - size_t confirmed_audio_bytes = 0; - size_t processed_chunks = 0; - bool chunked_stalled_on_speech = false; - - while (processed_chunks < kParakeetTdtMaxChunksPerProcess) { - const bool chunked_warmup = - handle->parakeet_committed_text.empty() && - handle->parakeet_tdt_pending_raw.empty(); - const size_t available_chunk_bytes = - handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes + right_context_bytes - ? handle->audio_buffer.size() - handle->parakeet_chunk_cursor_bytes - right_context_bytes - : 0; - const size_t chunk_decode_bytes = std::max( - encoder_frame_bytes, - ((chunked_warmup - ? available_chunk_bytes - : seconds_to_pcm_bytes(0.96)) / - encoder_frame_bytes) * encoder_frame_bytes); - if (handle->parakeet_chunk_cursor_bytes + chunk_decode_bytes + right_context_bytes > - handle->audio_buffer.size()) { - break; - } - - const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; - const size_t chunk_end_bytes = chunk_start_bytes + chunk_decode_bytes; - const size_t window_start_bytes = chunk_start_bytes > left_context_bytes - ? chunk_start_bytes - left_context_bytes - : 0; - const size_t window_end_bytes = chunk_end_bytes + right_context_bytes; - - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; - if (!run_parakeet_tdt_chunk_decode( - handle, - window_start_bytes, - window_end_bytes, - chunk_start_bytes, - chunk_end_bytes, - chunk_decode, - chunk_state)) { - last_error_message = "Chunked TDT decode failed."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - const bool chunk_silent = is_pcm_chunk_silent( - handle->audio_buffer.data() + chunk_start_bytes, - chunk_end_bytes - chunk_start_bytes, - 0.012f); - const std::string raw_chunk_text = - strip_unwanted_text_preserving_edges(chunk_decode.text); - const std::string raw_confirmed_text = - strip_unwanted_text_preserving_edges(chunk_decode.confirmed_text); - const std::string raw_pending_text = - strip_unwanted_text_preserving_edges(chunk_decode.pending_text); - - if (raw_chunk_text.empty() && !chunk_silent) { - chunked_stalled_on_speech = true; - break; - } - - if (!raw_chunk_text.empty()) { - std::string confirmed_delta = suppress_unwanted_text(raw_confirmed_text); - std::string pending_raw = raw_pending_text; - if (!confirmed_delta.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_delta, handle->custom_vocabulary); - } - handle->parakeet_tdt_pending_raw = pending_raw; - if (!confirmed_delta.empty()) { - if (chunk_decode.token_count > 0) { - handle->stream_total_tokens += static_cast(chunk_decode.token_count); - handle->stream_cumulative_tokens += static_cast(chunk_decode.token_count); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - parakeet_append_monotonic_text( - handle->parakeet_committed_text, - confirmed_delta); - parakeet_append_monotonic_text(confirmed, confirmed_delta); - const size_t confirmed_boundary_bytes = std::min( - chunk_end_bytes, - std::max( - chunk_start_bytes, - window_start_bytes + seconds_to_pcm_bytes(chunk_decode.confirmed_end_sec))); - if (confirmed_boundary_bytes > chunk_start_bytes) { - confirmed_audio_bytes += confirmed_boundary_bytes - chunk_start_bytes; - } - } - } - - handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; - handle->parakeet_tdt_decoder_state = chunk_state; - const size_t checkpoint_end_bytes = std::min( - chunk_end_bytes, - std::max( - window_start_bytes, - window_start_bytes + seconds_to_pcm_bytes(chunk_decode.resume_end_sec))); - while (!handle->parakeet_tdt_resume_checkpoints.empty() && - handle->parakeet_tdt_resume_checkpoints.back().end_bytes >= checkpoint_end_bytes) { - handle->parakeet_tdt_resume_checkpoints.pop_back(); - } - if (handle->parakeet_tdt_resume_checkpoints.empty() || - handle->parakeet_tdt_resume_checkpoints.back().end_bytes < checkpoint_end_bytes) { - handle->parakeet_tdt_resume_checkpoints.push_back({checkpoint_end_bytes, chunk_state}); - } - while (handle->parakeet_tdt_resume_checkpoints.size() > 16) { - handle->parakeet_tdt_resume_checkpoints.pop_front(); - } - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - static_cast( - handle->stream_audio_offset_sec + - pcm_bytes_to_seconds(checkpoint_end_bytes))); - ++processed_chunks; - } - - const bool fallback_to_legacy_tdt = - chunked_stalled_on_speech && - confirmed.empty() && - handle->parakeet_tdt_pending_raw.empty(); - if (fallback_to_legacy_tdt) { - const bool bootstrap_reset = - handle->parakeet_committed_text.empty() && - handle->previous_parakeet_pending.empty(); - reset_parakeet_tdt_chunk_stream(handle); - handle->parakeet_tdt_chunked_blocked_until_pause = true; - if (bootstrap_reset) { - handle->parakeet_chunk_cursor_bytes = 0; - } - } - - if (!fallback_to_legacy_tdt && - handle->parakeet_chunk_cursor_bytes > left_context_bytes) { - const size_t trim_bytes = - handle->parakeet_chunk_cursor_bytes - left_context_bytes; - handle->stream_audio_offset_sec += static_cast( - pcm_bytes_to_seconds(trim_bytes)); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + trim_bytes); - handle->parakeet_chunk_cursor_bytes -= trim_bytes; - while (!handle->parakeet_tdt_resume_checkpoints.empty() && - handle->parakeet_tdt_resume_checkpoints.front().end_bytes <= trim_bytes) { - handle->parakeet_tdt_resume_checkpoints.pop_front(); - } - for (auto& checkpoint : handle->parakeet_tdt_resume_checkpoints) { - checkpoint.end_bytes -= trim_bytes; - } - } - - if (!fallback_to_legacy_tdt) { - std::string pending_output; - std::string raw_json = "{}"; - if (!handle->parakeet_tdt_pending_raw.empty()) { - pending_output = suppress_unwanted_text(handle->parakeet_tdt_pending_raw); - if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_output, handle->custom_vocabulary); - } - } - if (handle->audio_buffer.size() > handle->parakeet_chunk_cursor_bytes) { - const size_t preview_start_bytes = handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0; - StreamWindowDecodeResult preview_decode; - if (run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + preview_start_bytes, - handle->audio_buffer.size() - preview_start_bytes, - preview_decode)) { - raw_json = preview_decode.raw_json; - if (pending_output.empty()) { - pending_output = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, - preview_decode.response); - if (!pending_output.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_output, handle->custom_vocabulary); - } - } - } - } - - handle->previous_parakeet_pending = pending_output; - handle->previous_parakeet_pending_ticks = pending_output.empty() ? 0 : 1; - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - handle->previous_segments.clear(); - handle->previous_cloud_handoff = false; - handle->parakeet_onset_active = false; - handle->parakeet_resume_guard_active = false; - - CloudResponse empty_cloud_result; - const std::string json_response = build_stream_response( - raw_json, - "", - confirmed, - pending_output, - "[]", - false, - pcm_bytes_to_seconds(confirmed_audio_bytes) * 1000.0, - 0, - 0, - empty_cloud_result, - handle->stream_first_token_seen ? handle->stream_first_token_ms : 0.0, - raw_json == "{}" ? handle->last_stream_total_time_ms : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tps : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, - raw_json == "{}" ? handle->last_stream_decode_tokens : -1.0, - raw_json == "{}" ? handle->last_stream_raw_decoder_tps : -1.0); - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - } - - auto decode_window = [&](size_t window_start_bytes, - size_t window_end_bytes, - StreamWindowDecodeResult& out, - float& out_offset_sec) -> bool { - if (window_end_bytes <= window_start_bytes || window_end_bytes > handle->audio_buffer.size()) { - return false; - } - out_offset_sec = static_cast( - handle->stream_audio_offset_sec + pcm_bytes_to_seconds(window_start_bytes)); - StreamWindowDecodeResult vad_decode; - const bool vad_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_vad_options_json, - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - vad_decode); - if (vad_ok && (!vad_decode.response.empty() || !vad_decode.segments.empty())) { - out = std::move(vad_decode); - return true; - } - - return run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + window_start_bytes, - window_end_bytes - window_start_bytes, - out); - }; - - constexpr size_t kParakeetHoldbackWords = 1; - constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence - constexpr size_t kParakeetStableFlushWords = 10; - constexpr size_t kParakeetStableFlushTicks = 2; - constexpr size_t kParakeetResumeGuardBytes = 24000; // ~0.75s speech - constexpr size_t kParakeetOnsetLeadSilenceBytes = 16000; // ~0.5s lead-in - constexpr size_t kParakeetOnsetMinConfirmWords = 6; - - const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); - const size_t prior_silence_run_bytes = handle->parakeet_silence_run_bytes; - const bool speech_resumed = - !chunk_silent && prior_silence_run_bytes >= kParakeetSilenceFlushBytes; - const size_t speech_chunk_start_bytes = - handle->audio_buffer.size() >= pcm_buffer_size - ? handle->audio_buffer.size() - pcm_buffer_size - : 0; - - if (chunk_silent) { - handle->parakeet_silence_run_bytes += pcm_buffer_size; - handle->parakeet_speech_run_bytes = 0; - if (handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes) { - handle->parakeet_resume_guard_active = true; - handle->parakeet_onset_active = false; - reset_parakeet_tdt_chunk_stream(handle); - const size_t retained_silence_bytes = std::min( - handle->parakeet_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); - handle->parakeet_onset_start_bytes = - handle->audio_buffer.size() > retained_silence_bytes - ? handle->audio_buffer.size() - retained_silence_bytes - : 0; - } - } else { - if (!handle->parakeet_onset_active && - handle->parakeet_committed_text.empty() && - handle->previous_parakeet_pending.empty()) { - handle->parakeet_onset_active = true; - handle->parakeet_onset_start_bytes = 0; - } - if (speech_resumed) { - handle->parakeet_resume_guard_active = true; - handle->parakeet_speech_run_bytes = 0; - handle->parakeet_onset_active = true; - const size_t retained_silence_bytes = std::min( - prior_silence_run_bytes, kParakeetOnsetLeadSilenceBytes); - handle->parakeet_onset_start_bytes = - speech_chunk_start_bytes > retained_silence_bytes - ? speech_chunk_start_bytes - retained_silence_bytes - : 0; - } - handle->parakeet_silence_run_bytes = 0; - if (handle->parakeet_resume_guard_active) { - handle->parakeet_speech_run_bytes += pcm_buffer_size; - if (handle->parakeet_speech_run_bytes >= kParakeetResumeGuardBytes) { - handle->parakeet_resume_guard_active = false; - handle->parakeet_speech_run_bytes = 0; - } - } - } - - const bool detected_pause = - handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; - const bool should_hold_cursor = handle->parakeet_onset_active; - - size_t processed_chunks = 0; - if (!should_hold_cursor) { - while (handle->parakeet_chunk_cursor_bytes + chunk_bytes + right_context_bytes <= - handle->audio_buffer.size() && - processed_chunks < kParakeetTdtMaxChunksPerProcess) { - handle->parakeet_chunk_cursor_bytes += chunk_bytes; - ++processed_chunks; - } - } - - if (!handle->parakeet_onset_active && - handle->parakeet_chunk_cursor_bytes > left_context_bytes) { - const size_t trim_bytes = - handle->parakeet_chunk_cursor_bytes - left_context_bytes; - handle->stream_audio_offset_sec += static_cast( - pcm_bytes_to_seconds(trim_bytes)); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + trim_bytes); - handle->parakeet_chunk_cursor_bytes -= trim_bytes; - } - - const bool onset_mode_active = handle->parakeet_onset_active; - const size_t preview_start_bytes = onset_mode_active - ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) - : (handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0); - const size_t preview_end_bytes = handle->audio_buffer.size(); - - StreamWindowDecodeResult response_decode; - float response_segments_offset_sec = 0.0f; - if (!decode_window(preview_start_bytes, preview_end_bytes, response_decode, response_segments_offset_sec)) { - last_error_message = "Transcription failed in stream process."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - std::string confirmed; - const std::string previous_pending = handle->previous_parakeet_pending; - std::string pending_tail = previous_pending; - const std::string normalized_response = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, response_decode.response); - if (!normalized_response.empty()) { - if (previous_pending.empty() || (onset_mode_active && handle->parakeet_resume_guard_active)) { - pending_tail = normalized_response; - } else { - const size_t stable_words = parakeet_common_prefix_words( - previous_pending, normalized_response); - const size_t confirm_words = - stable_words > kParakeetHoldbackWords - ? stable_words - kParakeetHoldbackWords - : 0; - confirmed = parakeet_take_first_words(normalized_response, confirm_words); - pending_tail = confirm_words > 0 - ? parakeet_drop_first_words(normalized_response, confirm_words) - : normalized_response; - } - } - - const bool onset_commit_too_small = - onset_mode_active && - !confirmed.empty() && - !detected_pause && - parakeet_word_count(confirmed) < kParakeetOnsetMinConfirmWords; - if (onset_commit_too_small) { - confirmed.clear(); - pending_tail = normalized_response; - } - - const bool pending_equivalent = - pending_tail == previous_pending || - (!pending_tail.empty() && - !previous_pending.empty() && - levenshtein_ci(pending_tail, previous_pending) <= 1); - const bool pending_changed = !pending_equivalent; - if (pending_changed) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - const bool stable_short_pending = - !pending_tail.empty() && - !handle->parakeet_resume_guard_active && - pending_equivalent && - handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && - parakeet_word_count(pending_tail) <= kParakeetStableFlushWords; - if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { - if (!confirmed.empty()) confirmed += " "; - confirmed += pending_tail; - pending_tail.clear(); - } - - double buffer_duration_ms = 0.0; - bool cloud_handoff_triggered = false; - uint64_t cloud_job_id = 0; - uint64_t cloud_result_job_id = 0; - CloudResponse cloud_result; - - if (!confirmed.empty()) { - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - std::string confirmed_delta = parakeet_emit_delta( - handle->parakeet_committed_text, confirmed); - if (!confirmed_delta.empty()) { - if (response_decode.decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(response_decode.decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(response_decode.decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - if (!handle->parakeet_committed_text.empty()) { - handle->parakeet_committed_text += " "; - } - handle->parakeet_committed_text += confirmed_delta; - confirmed = confirmed_delta; - if (onset_mode_active) { - handle->parakeet_onset_active = false; - handle->parakeet_onset_start_bytes = 0; - handle->parakeet_chunk_cursor_bytes = - handle->audio_buffer.size() > right_context_bytes - ? handle->audio_buffer.size() - right_context_bytes - : 0; - } - buffer_duration_ms = - static_cast(processed_chunks) * kParakeetTdtChunkSec * 1000.0; - } else { - confirmed.clear(); - } - } - - if (pending_tail.empty()) { - handle->previous_parakeet_pending_ticks = 0; - } else if (pending_equivalent) { - handle->previous_parakeet_pending_ticks = std::min( - handle->previous_parakeet_pending_ticks + 1, 1000000); - } else { - handle->previous_parakeet_pending_ticks = 1; - } - handle->previous_parakeet_pending = pending_tail; - if (pending_tail.empty()) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - std::vector current_segments = response_decode.segments; - const float segments_offset_sec = response_segments_offset_sec; - const std::string json_str = response_decode.raw_json; - const std::string pending_output = pending_tail; - handle->previous_cloud_handoff = false; - - for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { - if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { - CloudResponse job_result = it->result.get(); - CACTUS_LOG_INFO( - "cloud_handoff", - "Completed stream cloud job id=" << it->id - << " used_cloud=" << (job_result.used_cloud ? "true" : "false") - << " error=" << (job_result.error.empty() ? "none" : job_result.error) - << " transcript_chars=" << job_result.transcript.size()); - handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); - it = handle->pending_cloud_jobs.erase(it); - } else { - ++it; - } - } - - if (!handle->completed_cloud_results.empty()) { - cloud_result_job_id = handle->completed_cloud_results.front().first; - cloud_result = handle->completed_cloud_results.front().second; - handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); - - if (!cloud_result.api_key_hash.empty()) { - cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); - } - } - - constexpr int STREAM_TOKENS_CAP = 20000; - constexpr double STREAM_DURATION_CAP_MS = 600000.0; - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = - std::chrono::duration_cast( - now - handle->stream_start).count(); - if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || - elapsed_ms >= STREAM_DURATION_CAP_MS) { - double period_tps = (elapsed_ms > 0.0) - ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms - : 0.0; - - double cumulative_elapsed_ms = - std::chrono::duration_cast( - now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) - ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / - cumulative_elapsed_ms - : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - handle->stream_start = std::chrono::steady_clock::now(); - handle->stream_first_token_seen = false; - handle->stream_first_token_ms = 0.0; - handle->stream_total_tokens = 0; - } - - std::string json_response = build_stream_response( - json_str, - json_string(json_str, "error"), - confirmed, - pending_output, - serialize_segments_with_offset(current_segments, segments_offset_sec), - cloud_handoff_triggered, - buffer_duration_ms, - cloud_job_id, - cloud_result_job_id, - cloud_result - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription( - handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, - false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, - last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } - - const uint8_t* decode_pcm = handle->audio_buffer.data(); - size_t decode_pcm_size = handle->audio_buffer.size(); - if (is_parakeet_tdt) { - constexpr size_t kParakeetDecodeWindowBytes = 160000; // ~5.0s - if (decode_pcm_size > kParakeetDecodeWindowBytes) { - decode_pcm += (decode_pcm_size - kParakeetDecodeWindowBytes); - decode_pcm_size = kParakeetDecodeWindowBytes; - } - } - - cactus::telemetry::setStreamMode(true); - int result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - decode_pcm, - decode_pcm_size); - cactus::telemetry::setStreamMode(false); - if (result < 0) { - last_error_message = "Transcription failed in stream process."; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::string json_str(handle->transcribe_response_buffer); - std::string response = suppress_unwanted_text(json_string(json_str, "response")); - - cactus_reset(handle->model_handle); - - auto current_segments = parse_segments(json_str); - const float segments_offset_sec = static_cast(handle->stream_audio_offset_sec); - - std::string confirmed; - double buffer_duration_ms = 0.0; - bool cloud_handoff_triggered = false; - uint64_t cloud_job_id = 0; - uint64_t cloud_result_job_id = 0; - CloudResponse cloud_result; - double chunk_decode_tokens = json_number(json_str, "decode_tokens"); - if (chunk_decode_tokens < 0.0) { - chunk_decode_tokens = 0.0; - } - - if (is_parakeet_tdt) { - constexpr size_t kParakeetHoldbackWords = 1; - constexpr size_t kParakeetTailBytes = 64000; // ~2.0s left context - constexpr size_t kParakeetSilenceFlushBytes = 16000; // ~0.5s silence - constexpr size_t kParakeetStableFlushWords = 4; - constexpr size_t kParakeetStableFlushTicks = 2; - constexpr size_t kParakeetMediumTrimWords = 3; - constexpr size_t kParakeetAggressiveTrimWords = 6; - - const std::string previous_pending = handle->previous_parakeet_pending; - std::string pending_tail = previous_pending; - const bool chunk_silent = is_pcm_chunk_silent(pcm_buffer, pcm_buffer_size, 0.012f); - if (chunk_silent) { - handle->parakeet_silence_run_bytes += pcm_buffer_size; - } else { - handle->parakeet_silence_run_bytes = 0; - } - if (!response.empty()) { - const std::string normalized_response = parakeet_strip_recent_committed_prefix( - handle->parakeet_committed_text, response); - if (previous_pending.empty()) { - pending_tail = normalized_response; - } else { - const size_t stable_words = parakeet_common_prefix_words(previous_pending, normalized_response); - size_t confirm_words = 0; - if (stable_words > kParakeetHoldbackWords) { - confirm_words = stable_words - kParakeetHoldbackWords; - } - - confirmed = parakeet_take_first_words(normalized_response, confirm_words); - if (confirm_words > 0) { - pending_tail = parakeet_drop_first_words(normalized_response, confirm_words); - } else { - pending_tail = normalized_response; - } - } - } - - const bool pending_equivalent = - pending_tail == previous_pending || - (!pending_tail.empty() && - !previous_pending.empty() && - levenshtein_ci(pending_tail, previous_pending) <= 1); - const bool pending_changed = !pending_equivalent; - if (pending_changed) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - - const bool detected_pause = - handle->parakeet_silence_run_bytes >= kParakeetSilenceFlushBytes; - const bool stable_short_pending = - !pending_tail.empty() && - pending_equivalent && - handle->previous_parakeet_pending_ticks >= kParakeetStableFlushTicks && - parakeet_word_count(pending_tail) <= kParakeetStableFlushWords && - handle->audio_buffer.size() >= - (handle->previous_parakeet_audio_buffer_size + (kParakeetSilenceFlushBytes / 2)); - if ((detected_pause || stable_short_pending) && !pending_tail.empty()) { - if (!confirmed.empty()) confirmed += " "; - confirmed += pending_tail; - pending_tail.clear(); - } - - if (!confirmed.empty()) { - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - std::string confirmed_delta = parakeet_emit_delta( - handle->parakeet_committed_text, confirmed); - - const size_t max_trim = handle->audio_buffer.size() > kParakeetTailBytes - ? handle->audio_buffer.size() - kParakeetTailBytes - : 0; - const size_t confirmed_words = parakeet_word_count(confirmed); - const size_t hop_bytes = std::clamp( - pcm_buffer_size, static_cast(4000), static_cast(16000)); - size_t confirmed_bytes = std::min(hop_bytes, max_trim); - if (confirmed_words >= kParakeetAggressiveTrimWords || pending_tail.empty()) { - confirmed_bytes = max_trim; - } else if (confirmed_words >= kParakeetMediumTrimWords) { - confirmed_bytes = std::min(max_trim, hop_bytes * 2); - } - buffer_duration_ms = (confirmed_bytes / 2.0) / 16000.0 * 1000.0; - - if (!confirmed_delta.empty()) { - if (chunk_decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - if (!handle->parakeet_committed_text.empty()) { - handle->parakeet_committed_text += " "; - } - handle->parakeet_committed_text += confirmed_delta; - } - - if (confirmed_bytes > 0) { - if (handle->previous_cloud_handoff && !confirmed.empty()) { - cloud_handoff_triggered = true; - std::vector confirmed_audio( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); - std::string b64 = cloud_base64_encode(wav.data(), wav.size()); - cloud_job_id = handle->next_cloud_job_id++; - CACTUS_LOG_INFO( - "cloud_handoff", - "Queued stream cloud job id=" << cloud_job_id - << " confirmed_chars=" << confirmed.size() - << " audio_bytes=" << confirmed_audio.size()); - handle->pending_cloud_jobs.push_back({ - cloud_job_id, - std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) - }); - } - - handle->stream_audio_offset_sec += static_cast(confirmed_bytes) / (16000.0f * 2.0f); - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - } - - confirmed = confirmed_delta; - handle->previous_cloud_handoff = false; - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } else { - handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); - } - - if (pending_tail.empty()) { - handle->previous_parakeet_pending_ticks = 0; - } else if (pending_equivalent) { - handle->previous_parakeet_pending_ticks = std::min( - handle->previous_parakeet_pending_ticks + 1, 1000000); - } else { - handle->previous_parakeet_pending_ticks = 1; - } - handle->previous_parakeet_pending = pending_tail; - if (pending_tail.empty()) { - handle->previous_parakeet_audio_buffer_size = handle->audio_buffer.size(); - } - handle->previous_segments = current_segments; - } else { - size_t confirmed_segments = 0; - float confirmed_end_sec = 0.0f; - for (size_t i = 0; i < std::min(handle->previous_segments.size(), current_segments.size()); ++i) { - if (handle->previous_segments[i].text != current_segments[i].text) { - break; - } - confirmed_segments = i + 1; - confirmed_end_sec = std::min(handle->previous_segments[i].end, current_segments[i].end); - } - - if (confirmed_segments > 0 && confirmed_end_sec > 0.0f) { - buffer_duration_ms = confirmed_end_sec * 1000.0; - - for (size_t i = 0; i < confirmed_segments; ++i) { - if (!confirmed.empty()) confirmed += ' '; - confirmed += handle->previous_segments[i].text; - } - - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed, handle->custom_vocabulary); - } - - if (chunk_decode_tokens > 0.0) { - handle->stream_total_tokens += static_cast(std::round(chunk_decode_tokens)); - handle->stream_cumulative_tokens += static_cast(std::round(chunk_decode_tokens)); - } - - if (!handle->stream_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_first_token_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - handle->stream_first_token_seen = true; - } - - if (!handle->stream_session_first_token_seen) { - auto now = std::chrono::steady_clock::now(); - handle->stream_session_first_token_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - handle->stream_session_first_token_seen = true; - } - - const size_t confirmed_bytes = std::min( - static_cast(confirmed_end_sec * 16000.0) * 2, - handle->audio_buffer.size() - ); - - if (handle->previous_cloud_handoff && !confirmed.empty()) { - cloud_handoff_triggered = true; - std::vector confirmed_audio( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - auto wav = cloud_build_wav(confirmed_audio.data(), confirmed_audio.size()); - std::string b64 = cloud_base64_encode(wav.data(), wav.size()); - cloud_job_id = handle->next_cloud_job_id++; - CACTUS_LOG_INFO( - "cloud_handoff", - "Queued stream cloud job id=" << cloud_job_id - << " confirmed_chars=" << confirmed.size() - << " audio_bytes=" << confirmed_audio.size()); - handle->pending_cloud_jobs.push_back({ - cloud_job_id, - std::async(std::launch::async, cloud_transcribe_request, b64, confirmed, 15L, nullptr) - }); - } - - handle->stream_audio_offset_sec += confirmed_end_sec; - handle->audio_buffer.erase( - handle->audio_buffer.begin(), - handle->audio_buffer.begin() + confirmed_bytes - ); - constexpr float kSegmentEpsilonSec = 0.02f; - size_t current_tail_start = 0; - while (current_tail_start < current_segments.size() && - current_segments[current_tail_start].end <= confirmed_end_sec + kSegmentEpsilonSec) { - ++current_tail_start; - } - handle->previous_segments = std::vector( - current_segments.begin() + current_tail_start, current_segments.end()); - handle->previous_cloud_handoff = false; - } else { - handle->previous_segments = current_segments; - handle->previous_cloud_handoff = json_bool(json_str, "cloud_handoff"); - } - } - - for (auto it = handle->pending_cloud_jobs.begin(); it != handle->pending_cloud_jobs.end(); ) { - if (it->result.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) { - CloudResponse job_result = it->result.get(); - CACTUS_LOG_INFO( - "cloud_handoff", - "Completed stream cloud job id=" << it->id - << " used_cloud=" << (job_result.used_cloud ? "true" : "false") - << " error=" << (job_result.error.empty() ? "none" : job_result.error) - << " transcript_chars=" << job_result.transcript.size()); - handle->completed_cloud_results.push_back({it->id, std::move(job_result)}); - it = handle->pending_cloud_jobs.erase(it); - } else { - ++it; - } - } - - if (!handle->completed_cloud_results.empty()) { - cloud_result_job_id = handle->completed_cloud_results.front().first; - cloud_result = handle->completed_cloud_results.front().second; - handle->completed_cloud_results.erase(handle->completed_cloud_results.begin()); - - if (!cloud_result.api_key_hash.empty()) { - cactus::telemetry::setCloudKey(cloud_result.api_key_hash.c_str()); - } - } - - constexpr int STREAM_TOKENS_CAP = 20000; - constexpr double STREAM_DURATION_CAP_MS = 600000.0; - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - if (handle->stream_total_tokens >= STREAM_TOKENS_CAP || elapsed_ms >= STREAM_DURATION_CAP_MS) { - double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; - - double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - handle->stream_start = std::chrono::steady_clock::now(); - handle->stream_first_token_seen = false; - handle->stream_first_token_ms = 0.0; - handle->stream_total_tokens = 0; - } - - std::string pending_output = response; - if (is_parakeet_tdt && !handle->previous_parakeet_pending.empty()) { - pending_output = handle->previous_parakeet_pending; - } - - std::string json_response = build_stream_response( - json_str, - json_string(json_str, "error"), - confirmed, - pending_output, - serialize_segments_with_offset(current_segments, segments_offset_sec), - cloud_handoff_triggered, - buffer_duration_ms, - cloud_job_id, - cloud_result_job_id, - cloud_result - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, last_error_message.c_str()); - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - return static_cast(json_response.length()); - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_process: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, e.what()); - return -1; - } catch (...) { - last_error_message = "Unknown exception during stream transcription processing"; - CACTUS_LOG_ERROR("stream_transcribe_process", last_error_message); - handle_error_response("Unknown error during stream processing", response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, 0.0, 0.0, 0, "Unknown error during stream processing"); - return -1; - } -} - -int cactus_stream_transcribe_stop( - cactus_stream_transcribe_t stream, - char* response_buffer, - size_t buffer_size -) { - if (!stream) { - last_error_message = "Stream not initialized."; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - return -1; - } - - auto* handle = static_cast(stream); - auto clear_stream_vocab_bias = [&]() { - if (handle->has_custom_vocabulary_bias && - handle->model_handle && - handle->model_handle->model) { - handle->model_handle->model->clear_vocab_bias(); - } - }; - - if (!response_buffer || buffer_size == 0) { - clear_stream_vocab_bias(); - delete handle; - return 0; - } - - try { - const auto model_type = handle->model_handle->model->get_config().model_type; - bool is_moonshine = model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = - model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet_ctc = - model_type == cactus::engine::Config::ModelType::PARAKEET; - bool is_parakeet = - is_parakeet_ctc || is_parakeet_tdt; - bool is_tinyllama = model_type == cactus::engine::Config::ModelType::GEMMA4; - - std::string final_confirmed; - if (is_parakeet_tdt) { - final_confirmed = handle->parakeet_committed_text; - if (!(handle->parakeet_tdt_chunked_stream && handle->parakeet_tdt_decode_context.initialized) && - !handle->previous_parakeet_pending.empty()) { - const std::string normalized_pending = parakeet_strip_recent_committed_prefix( - final_confirmed, handle->previous_parakeet_pending); - const std::string pending_delta = parakeet_emit_delta( - final_confirmed, normalized_pending); - if (!pending_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += pending_delta; - } - } - } else { - for (const auto& seg : handle->previous_segments) { - if (!final_confirmed.empty()) final_confirmed += ' '; - final_confirmed += seg.text; - } - } - - if (is_parakeet_tdt && - handle->parakeet_tdt_chunked_stream && - handle->parakeet_tdt_decode_context.initialized) { - auto& ctx = handle->parakeet_tdt_decode_context; - auto* tdt_model = static_cast( - handle->model_handle->model.get()); - const size_t total_samples = ctx.audio_samples.size(); - if (total_samples > ctx.samples_decoded_up_to) { - constexpr size_t kTdtLeftContextSamples = 8 * 16000; - const size_t window_start_sample = ctx.samples_decoded_up_to > kTdtLeftContextSamples - ? ctx.samples_decoded_up_to - kTdtLeftContextSamples - : 0; - std::vector window_audio( - ctx.audio_samples.begin() + window_start_sample, - ctx.audio_samples.end()); - - auto cfg = get_parakeet_spectrogram_config(); - const size_t waveform_samples = window_audio.size(); - apply_preemphasis(window_audio, 0.97f); - window_audio = ctx.audio_processor.compute_spectrogram(window_audio, cfg); - normalize_parakeet_log_mel(window_audio, ctx.mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(window_audio, ctx.mel_bins, valid_frames); - - const uint32_t subsampling = - std::max(1, handle->model_handle->model->get_config().subsampling_factor); - const size_t samples_per_enc_frame = cfg.hop_length * subsampling; - const size_t context_samples = ctx.samples_decoded_up_to - window_start_sample; - const size_t decode_start_frame = context_samples / samples_per_enc_frame; - const size_t total_window_samples = total_samples - window_start_sample; - const size_t decode_end_frame = total_window_samples / samples_per_enc_frame; - - if (decode_end_frame > decode_start_frame) { - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - { - std::lock_guard lock(handle->model_handle->model_mutex); - cactus::telemetry::setStreamMode(true); - try { - chunk_decode = tdt_model->decode_chunk_stream( - window_audio, - decode_start_frame, - decode_start_frame, - decode_end_frame, - ctx.decoder_state); - cactus_reset(handle->model_handle); - } catch (...) { - cactus::telemetry::setStreamMode(false); - throw; - } - cactus::telemetry::setStreamMode(false); - } - - std::string confirmed_flush = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_flush = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_flush, handle->custom_vocabulary); - } - if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_flush, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, confirmed_flush); - parakeet_append_monotonic_text(final_confirmed, pending_flush); - } - } else if (!handle->previous_parakeet_pending.empty()) { - parakeet_append_monotonic_text(final_confirmed, handle->previous_parakeet_pending); - } - } - - if (!handle->audio_buffer.empty()) { - std::string whisper_prompt = "<|startoftranscript|><|" + handle->options.language + "|><|transcribe|><|notimestamps|>"; - const char* transcribe_prompt = - is_tinyllama ? "Transcribe the audio." : - (is_moonshine || is_parakeet) ? "" : - whisper_prompt.c_str(); - std::string effective_transcribe_options_json = handle->transcribe_options_json; - if (is_parakeet && !is_parakeet_tdt) { - effective_transcribe_options_json = ensure_json_bool_option( - effective_transcribe_options_json, "use_vad", false); - } - - if (is_parakeet_tdt) { - const bool use_chunked_stop = - handle->parakeet_tdt_chunked_stream && - !handle->parakeet_tdt_chunked_blocked_until_pause && - (handle->parakeet_tdt_decoder_state.initialized || - !handle->parakeet_tdt_resume_checkpoints.empty() || - !handle->parakeet_tdt_pending_raw.empty()); - if (use_chunked_stop) { - final_confirmed = handle->parakeet_committed_text; - constexpr double kParakeetTdtLeftContextSec = 2.0; - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - if (handle->parakeet_chunk_cursor_bytes < handle->audio_buffer.size()) { - const size_t chunk_start_bytes = handle->parakeet_chunk_cursor_bytes; - const size_t chunk_end_bytes = handle->audio_buffer.size(); - const size_t window_start_bytes = chunk_start_bytes > left_context_bytes - ? chunk_start_bytes - left_context_bytes - : 0; - cactus::engine::ParakeetTDTModel::ChunkStreamResult chunk_decode; - cactus::engine::ParakeetTDTModel::ChunkStreamState chunk_state; - if (run_parakeet_tdt_chunk_decode( - handle, - window_start_bytes, - chunk_end_bytes, - chunk_start_bytes, - chunk_end_bytes, - chunk_decode, - chunk_state)) { - std::string confirmed_chunk = suppress_unwanted_text(chunk_decode.confirmed_text); - std::string pending_chunk = suppress_unwanted_text(chunk_decode.pending_text); - if (!confirmed_chunk.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(confirmed_chunk, handle->custom_vocabulary); - } - if (!pending_chunk.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(pending_chunk, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, confirmed_chunk); - handle->parakeet_tdt_pending_raw = - strip_unwanted_text_preserving_edges(chunk_decode.pending_text); - handle->parakeet_chunk_cursor_bytes = chunk_end_bytes; - handle->parakeet_tdt_decoder_state = chunk_state; - handle->parakeet_committed_until_sec = std::max( - handle->parakeet_committed_until_sec, - static_cast( - handle->stream_audio_offset_sec + - pcm_bytes_to_seconds( - std::min( - chunk_end_bytes, - std::max( - window_start_bytes, - window_start_bytes + seconds_to_pcm_bytes( - chunk_decode.resume_end_sec)))))); - } - } else if (!handle->previous_parakeet_pending.empty()) { - const std::string normalized_pending = parakeet_strip_recent_committed_prefix( - final_confirmed, handle->previous_parakeet_pending); - const std::string pending_delta = parakeet_emit_delta( - final_confirmed, normalized_pending); - if (!pending_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += pending_delta; - } - } - if (!handle->parakeet_tdt_pending_raw.empty()) { - std::string pending_flush = suppress_unwanted_text( - handle->parakeet_tdt_pending_raw); - if (!pending_flush.empty() && !handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction( - pending_flush, handle->custom_vocabulary); - } - parakeet_append_monotonic_text(final_confirmed, pending_flush); - handle->parakeet_tdt_pending_raw.clear(); - } - handle->parakeet_committed_text = final_confirmed; - } else { - constexpr double kParakeetTdtLeftContextSec = 2.0; - constexpr float kParakeetTdtCommitEpsilonSec = 0.03f; - const size_t left_context_bytes = seconds_to_pcm_bytes(kParakeetTdtLeftContextSec); - const size_t flush_start_bytes = handle->parakeet_onset_active - ? std::min(handle->parakeet_onset_start_bytes, handle->audio_buffer.size()) - : (handle->parakeet_chunk_cursor_bytes > left_context_bytes - ? handle->parakeet_chunk_cursor_bytes - left_context_bytes - : 0); - const std::string tdt_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", true); - const std::string tdt_no_vad_options_json = ensure_json_bool_option( - handle->transcribe_options_json, "use_vad", false); - - StreamWindowDecodeResult flush_decode; - bool flush_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_vad_options_json, - handle->audio_buffer.data() + flush_start_bytes, - handle->audio_buffer.size() - flush_start_bytes, - flush_decode); - if (flush_ok && flush_decode.response.empty() && flush_decode.segments.empty()) { - flush_ok = run_stream_window_transcribe( - handle, - transcribe_prompt, - tdt_no_vad_options_json, - handle->audio_buffer.data() + flush_start_bytes, - handle->audio_buffer.size() - flush_start_bytes, - flush_decode); - } - if (flush_ok) { - const float flush_offset_sec = static_cast( - handle->stream_audio_offset_sec + pcm_bytes_to_seconds(flush_start_bytes)); - float flushed_until_sec = handle->parakeet_committed_until_sec; - std::string flushed_text = join_segments_by_end_time( - flush_decode.segments, - flush_offset_sec, - handle->parakeet_committed_until_sec + kParakeetTdtCommitEpsilonSec, - std::numeric_limits::max(), - &flushed_until_sec); - if (flushed_text.empty()) { - flushed_text = parakeet_strip_recent_committed_prefix( - final_confirmed, flush_decode.response); - } - const std::string effective_flush_delta = - parakeet_emit_delta(final_confirmed, flushed_text); - if (!effective_flush_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += effective_flush_delta; - } - if (flushed_until_sec > handle->parakeet_committed_until_sec) { - handle->parakeet_committed_until_sec = flushed_until_sec; - } - } else { - CACTUS_LOG_WARN( - "stream_transcribe_stop", - "Final flush transcription failed; returning previously confirmed transcript."); - } - } - } else { - cactus::telemetry::setStreamMode(true); - const int flush_result = cactus_transcribe( - handle->model_handle, - nullptr, - transcribe_prompt, - handle->transcribe_response_buffer, - sizeof(handle->transcribe_response_buffer), - effective_transcribe_options_json.empty() ? nullptr : effective_transcribe_options_json.c_str(), - nullptr, - nullptr, - handle->audio_buffer.data(), - handle->audio_buffer.size()); - cactus::telemetry::setStreamMode(false); - - cactus_reset(handle->model_handle); - - if (flush_result >= 0) { - std::string flush_json(handle->transcribe_response_buffer); - std::string flushed_text = suppress_unwanted_text(json_string(flush_json, "response")); - if (flushed_text.empty()) { - auto flush_segments = parse_segments(flush_json); - for (const auto& seg : flush_segments) { - if (!flushed_text.empty()) flushed_text += ' '; - flushed_text += seg.text; - } - } - if (!flushed_text.empty()) { - if (is_parakeet_tdt) { - const std::string normalized_flush = parakeet_strip_recent_committed_prefix( - final_confirmed, flushed_text); - const std::string effective_flush_delta = - parakeet_emit_delta(final_confirmed, normalized_flush); - if (!effective_flush_delta.empty()) { - if (!final_confirmed.empty()) final_confirmed += " "; - final_confirmed += effective_flush_delta; - } - } else { - final_confirmed = flushed_text; - } - } - } else { - CACTUS_LOG_WARN( - "stream_transcribe_stop", - "Final flush transcription failed; returning previously confirmed segments."); - } - } - } - - if (!handle->custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(final_confirmed, handle->custom_vocabulary); - } - - std::string json_response = "{\"success\":true,\"confirmed\":\"" + - escape_json(final_confirmed) + "\"}"; - - auto now = std::chrono::steady_clock::now(); - double elapsed_ms = std::chrono::duration_cast(now - handle->stream_start).count(); - double period_tps = (elapsed_ms > 0.0) ? (static_cast(handle->stream_total_tokens) * 1000.0) / elapsed_ms : 0.0; - - double cumulative_elapsed_ms = std::chrono::duration_cast(now - handle->stream_session_start).count(); - double cumulative_tps = (cumulative_elapsed_ms > 0.0) ? (static_cast(handle->stream_cumulative_tokens) * 1000.0) / cumulative_elapsed_ms : 0.0; - - cactus::telemetry::recordStreamTranscription( - handle->model_handle->model_name.c_str(), - true, - handle->stream_first_token_ms, - period_tps, - elapsed_ms, - handle->stream_total_tokens, - handle->stream_session_first_token_ms, - cumulative_tps, - cumulative_elapsed_ms, - handle->stream_cumulative_tokens, - "" - ); - - if (json_response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, last_error_message.c_str()); - clear_stream_vocab_bias(); - delete handle; - return -1; - } - - std::strcpy(response_buffer, json_response.c_str()); - clear_stream_vocab_bias(); - delete handle; - return static_cast(json_response.length()); - } catch (const std::exception& e) { - last_error_message = "Exception during stream_transcribe_stop: " + std::string(e.what()); - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, e.what()); - clear_stream_vocab_bias(); - delete handle; - return -1; - } catch (...) { - last_error_message = "Unknown exception during stream transcription stop"; - CACTUS_LOG_ERROR("stream_transcribe_stop", last_error_message); - handle_error_response("Unknown error during stream stop", response_buffer, buffer_size); - cactus::telemetry::recordStreamTranscription(handle->model_handle ? handle->model_handle->model_name.c_str() : nullptr, false, handle->stream_first_token_ms, 0.0, 0.0, handle->stream_total_tokens, handle->stream_session_first_token_ms, 0.0, 0.0, handle->stream_cumulative_tokens, "Unknown error during stream stop"); - clear_stream_vocab_bias(); - delete handle; - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_telemetry.cpp b/cactus/ffi/cactus_telemetry.cpp deleted file mode 100644 index 1844f0e22..000000000 --- a/cactus/ffi/cactus_telemetry.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "cactus_ffi.h" -#include "telemetry/telemetry.h" - -extern "C" { - -void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version) { - cactus::telemetry::setTelemetryEnvironment(framework, cache_location, version); -} - -void cactus_set_app_id(const char* app_id) { - cactus::telemetry::setAppId(app_id); -} - -void cactus_telemetry_flush(void) { - cactus::telemetry::flush(); -} - -void cactus_telemetry_shutdown(void) { - cactus::telemetry::shutdown(); -} - -} diff --git a/cactus/ffi/cactus_transcribe.cpp b/cactus/ffi/cactus_transcribe.cpp deleted file mode 100644 index 539c0f155..000000000 --- a/cactus/ffi/cactus_transcribe.cpp +++ /dev/null @@ -1,829 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "telemetry/telemetry.h" -#include "../../libs/audio/wav.h" -#include -#include -#include -#include -#include -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; -using cactus::audio::WHISPER_TARGET_FRAMES; -using cactus::audio::WHISPER_SAMPLE_RATE; -using cactus::audio::apply_preemphasis; -using cactus::audio::get_parakeet_spectrogram_config; -using cactus::audio::get_whisper_spectrogram_config; -using cactus::audio::init_whisper_mel_filters; -using cactus::audio::normalize_parakeet_log_mel; -using cactus::audio::normalize_whisper_mel; -using cactus::audio::trim_mel_frames; -using cactus::audio::get_htk_spectrogram_config; -using cactus::audio::get_gemma4_audio_spectrogram_config; -using cactus::audio::transpose_mel_to_frame_major; - -static constexpr size_t WHISPER_MAX_DECODER_POSITIONS = 448; -static constexpr size_t MAX_CHUNK_SAMPLES = WHISPER_SAMPLE_RATE * 30; -static constexpr size_t MAX_CONTEXT_WORDS = 64; - -static std::string extract_whisper_language_code(std::string token_text) { - const size_t open = token_text.find("<|"); - if (open == std::string::npos) return ""; - - const size_t close = token_text.find("|>", open + 2); - if (close == std::string::npos || close <= open + 2) return ""; - - std::string inner = token_text.substr(open + 2, close - (open + 2)); - std::transform(inner.begin(), inner.end(), inner.begin(), [](unsigned char c) { - return static_cast(std::tolower(c)); - }); - - static const std::vector non_language_tokens = { - "startoftranscript", "transcribe", "translate", "notimestamps", - "endoftext", "nospeech", "nocaptions", "sot", "sot_prev", "sot_lm" - }; - for (const auto& reserved : non_language_tokens) { - if (inner == reserved) return ""; - } - - if (inner.size() < 2 || inner.size() > 8) return ""; - for (char c : inner) { - const bool ok = (c >= 'a' && c <= 'z') || c == '-'; - if (!ok) return ""; - } - return inner; -} - -static bool is_terminal_transcription_piece(const std::string& piece) { - return piece == "<|endoftext|>" || - piece == "<|endoftranscript|>" || - piece == "" || - piece == ""; -} - -extern "C" { - -int cactus_transcribe( - cactus_model_t model, - const char* audio_file_path, - const char* prompt, - char* response_buffer, - size_t buffer_size, - const char* options_json, - cactus_token_callback callback, - void* user_data, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("transcribe", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - std::lock_guard lock(handle->model_mutex); - handle->should_stop = false; - - float cloud_handoff_threshold = handle->model->get_config().default_cloud_handoff_threshold; - const std::string opts = options_json ? options_json : ""; - InferenceOptions options = parse_inference_options_json(opts); - { - size_t pos = opts.find("\"cloud_handoff_threshold\""); - if (pos != std::string::npos) { - pos = opts.find(':', pos); - if (pos != std::string::npos) { - ++pos; - while (pos < opts.size() && std::isspace(static_cast(opts[pos]))) ++pos; - try { - cloud_handoff_threshold = std::stof(opts.c_str() + pos); - } catch (...) {} - } - } - } - - const char* force_handoff_env = std::getenv("CACTUS_FORCE_HANDOFF"); - if (force_handoff_env && force_handoff_env[0] == '1' && force_handoff_env[1] == '\0') { - cloud_handoff_threshold = 0.0001f; - } - - const bool request_has_custom_vocabulary_options = - opts.find("\"custom_vocabulary\"") != std::string::npos || - opts.find("\"vocabulary_boost\"") != std::string::npos; - const bool apply_request_scoped_vocabulary_bias = - request_has_custom_vocabulary_options && !handle->model->has_vocab_bias(); - - struct ScopedVocabularyBiasReset { - Model* model; - bool clear_on_exit; - ~ScopedVocabularyBiasReset() { - if (clear_on_exit && model) { - model->clear_vocab_bias(); - } - } - } scoped_vocabulary_bias_reset{ - handle->model.get(), - apply_request_scoped_vocabulary_bias - }; - std::vector custom_vocabulary; - float vocabulary_boost_unused = 5.0f; - parse_custom_vocabulary_options(opts, custom_vocabulary, vocabulary_boost_unused); - - if (apply_request_scoped_vocabulary_bias) { - apply_custom_vocabulary_options(handle->model.get(), opts); - } - - if (request_has_custom_vocabulary_options && - options.temperature == 0.0f && options.top_p <= 0.0f && options.top_k == 0) { - // Keep deterministic decoding while ensuring bias is applied in sampling. - options.top_k = 1; - } - - bool is_whisper = handle->model->get_config().model_type == cactus::engine::Config::ModelType::WHISPER; - bool is_moonshine = handle->model->get_config().model_type == cactus::engine::Config::ModelType::MOONSHINE; - bool is_parakeet_tdt = handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_parakeet = - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET || - handle->model->get_config().model_type == cactus::engine::Config::ModelType::PARAKEET_TDT; - bool is_gemma4 = handle->model->get_config().model_type == cactus::engine::Config::ModelType::GEMMA4; - - std::vector audio_samples; - if (audio_file_path == nullptr) { - auto waveform_fp32 = cactus::audio::pcm_buffer_to_float_samples(pcm_buffer, pcm_buffer_size); - audio_samples = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); - } else { - AudioFP32 audio = load_wav(audio_file_path); - audio_samples = resample_to_16k_fp32(audio.samples, audio.sample_rate); - } - - if (opts.find("\"max_tokens\"") == std::string::npos) { - const float audio_length_sec = static_cast(audio_samples.size()) / static_cast(WHISPER_SAMPLE_RATE); - const float tps = is_parakeet ? 30.0f : (is_gemma4 ? 30.0f : 20.0f); - const size_t estimated = static_cast(audio_length_sec * tps); - options.max_tokens = std::max(estimated, 100); - } - - if (is_gemma4) { - if (audio_samples.empty()) { - handle_error_response("No audio input provided", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "No audio input"); - return -1; - } - - const auto& model_config = handle->model->get_config(); - uint32_t audio_token_id = model_config.audio_token_id; - if (audio_token_id == 0) { - CACTUS_LOG_WARN("transcribe", "audio_token_id not set in config, using default 258881"); - audio_token_id = 258881; - } - - auto audio_prep = cactus::audio::preprocess_audio_for_gemma4(audio_samples, model_config); - std::vector& audio_features = audio_prep.features; - size_t num_soft_tokens = audio_prep.num_soft_tokens; - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); - return -1; - } - - std::string task_text = (prompt[0] != '\0') ? std::string(prompt) : "Transcribe the audio."; - auto prefix_tokens = tokenizer->encode("<|turn>user\n" + task_text + "<|audio>"); - auto suffix_tokens = tokenizer->encode("\n<|turn>model\n"); - - std::vector tokens; - tokens.reserve(prefix_tokens.size() + num_soft_tokens + suffix_tokens.size()); - tokens.insert(tokens.end(), prefix_tokens.begin(), prefix_tokens.end()); - for (size_t j = 0; j < num_soft_tokens; j++) - tokens.push_back(audio_token_id); - tokens.insert(tokens.end(), suffix_tokens.begin(), suffix_tokens.end()); - - std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; - auto append_stop = [&](const char* stop_text) { - std::vector seq = tokenizer->encode(stop_text); - if (!seq.empty()) - stop_token_sequences.push_back(std::move(seq)); - }; - append_stop(""); - append_stop(""); - append_stop(""); - - const size_t prompt_token_count = tokens.size(); - double time_to_first_token = 0.0; - size_t completion_tokens = 0; - std::string final_text; - float total_entropy_sum = 0.0f; - float max_token_entropy_norm = 0.0f; - std::vector generated_tokens; - generated_tokens.reserve(options.max_tokens); - - for (size_t i = 0; i < options.max_tokens; ++i) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - uint32_t next_token = handle->model->decode_with_audio( - tokens, audio_features, - options.temperature, options.top_p, options.top_k, - "", &token_entropy, - options.min_p, options.repetition_penalty - ); - - if (completion_tokens == 0) [[unlikely]] { - auto t_first = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; - } - - total_entropy_sum += token_entropy; - if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; - - generated_tokens.emplace_back(next_token); - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) - break; - - std::string piece = tokenizer->decode({ next_token }); - tokens.emplace_back(next_token); - completion_tokens++; - final_text += piece; - if (callback) callback(piece.c_str(), next_token, user_data); - } - - cactus_reset(model); - - float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; - float confidence = 1.0f - mean_entropy; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - double prefill_tps = time_to_first_token > 0 ? (prompt_token_count * 1000.0) / time_to_first_token : 0.0; - double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - if (!final_text.empty() && final_text[0] == ' ') - final_text.erase(0, 1); - - // Strip thinking channel content: <|channel>thought... - { - auto chan_start = final_text.find("<|channel>"); - auto chan_end = final_text.find(""); - if (chan_start != std::string::npos && chan_end != std::string::npos && chan_end > chan_start) { - size_t after = chan_end + std::string("").length(); - final_text = final_text.substr(after); - if (!final_text.empty() && final_text[0] == ' ') - final_text.erase(0, 1); - } - } - - const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && - cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; - - std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_token_count, completion_tokens, confidence, cloud_handoff); - - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); - std::strcpy(response_buffer, json.c_str()); - return static_cast(json.size()); - } - - auto to_sec = [](size_t samples) { - return static_cast(samples) / static_cast(WHISPER_SAMPLE_RATE); - }; - - struct AudioChunk { - std::vector audio; - float start_sec; - float end_sec; - std::vector> anchors; - }; - std::vector audio_chunks; - - if (options.use_vad) { - auto* vad = static_cast(handle->vad_model.get()); - auto vad_segments = vad->get_speech_timestamps(audio_samples, {}); - audio_chunks.reserve(vad_segments.size()); - - std::vector current; - std::vector> current_anchors; - size_t chunk_start_sample = 0; - size_t chunk_end_sample = 0; - float concat_cursor = 0.0f; - for (const auto& seg : vad_segments) { - size_t end = std::min(seg.end, audio_samples.size()); - if (current.size() + (end - seg.start) > MAX_CHUNK_SAMPLES) { - audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); - current.clear(); - current_anchors.clear(); - concat_cursor = 0.0f; - } - if (current.empty()) chunk_start_sample = seg.start; - chunk_end_sample = end; - current_anchors.emplace_back(concat_cursor, to_sec(seg.start)); - concat_cursor += to_sec(end - seg.start); - current.insert( - current.end(), - audio_samples.begin() + seg.start, - audio_samples.begin() + end - ); - } - - if (!current.empty()) { - audio_chunks.emplace_back(std::move(current), to_sec(chunk_start_sample), to_sec(chunk_end_sample), std::move(current_anchors)); - } - - if (audio_chunks.empty()) { - CACTUS_LOG_DEBUG("transcribe", "VAD detected only silence, returning empty transcription"); - auto vad_end_time = std::chrono::high_resolution_clock::now(); - double vad_total_time_ms = std::chrono::duration_cast(vad_end_time - start_time).count() / 1000.0; - std::string json = construct_response_json("", {}, 0.0, vad_total_time_ms, 0.0, 0.0, 0, 0, 1.0f); - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, 0.0, 0.0, vad_total_time_ms, 0, get_ram_usage_mb(), ""); - std::strcpy(response_buffer, json.c_str()); - return static_cast(json.size()); - } - } else { - audio_chunks.reserve((audio_samples.size() + MAX_CHUNK_SAMPLES - 1) / MAX_CHUNK_SAMPLES); - for (size_t start = 0; start < audio_samples.size(); start += MAX_CHUNK_SAMPLES) { - size_t end = std::min(start + MAX_CHUNK_SAMPLES, audio_samples.size()); - audio_chunks.emplace_back(std::vector(audio_samples.begin() + start, audio_samples.begin() + end), to_sec(start), to_sec(end)); - } - } - - auto cfg = is_parakeet ? get_parakeet_spectrogram_config() : get_whisper_spectrogram_config(); - size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); - const bool is_whisper_v3 = is_whisper && mel_bins > 80; - AudioProcessor ap; - if (is_parakeet) { - ap.init_mel_filters(cfg.n_fft / 2 + 1, mel_bins, 0.0f, 8000.0f, WHISPER_SAMPLE_RATE); - } else if (!is_moonshine) { - init_whisper_mel_filters(ap, cfg, mel_bins); - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - CACTUS_LOG_ERROR("transcribe", "Tokenizer unavailable"); - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Tokenizer unavailable"); - return -1; - } - - std::vector initial_tokens = tokenizer->encode( - prompt && prompt[0] != '\0' - ? std::string(prompt) - : (is_whisper ? "<|startoftranscript|>" : std::string()) - ); - if (initial_tokens.empty() && !is_moonshine && !is_parakeet) { - CACTUS_LOG_ERROR("transcribe", "Decoder input tokens empty after encoding prompt"); - handle_error_response("Decoder input tokens empty", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Decoder input tokens empty"); - return -1; - } - - float max_tps = handle->model->get_config().default_max_tps; - if (max_tps < 0) max_tps = 100; - - std::vector> stop_token_sequences = {{ tokenizer->get_eos_token() }}; - auto append_exact_stop_sequence = [&](const char* stop_text) { - std::vector seq = tokenizer->encode(stop_text); - if (!seq.empty() && tokenizer->decode(seq) == stop_text) { - stop_token_sequences.push_back(std::move(seq)); - } - }; - append_exact_stop_sequence("<|endoftext|>"); - append_exact_stop_sequence("<|endoftranscript|>"); - append_exact_stop_sequence(""); - append_exact_stop_sequence(""); - - double time_to_first_token = 0.0; - size_t completion_tokens = 0; - std::string final_text; - float total_entropy_sum = 0.0f; - float max_token_entropy_norm = 0.0f; - - const std::regex whisper_timestamp_re(R"(<\|(\d+(?:\.\d+)?)\|>)"); - std::vector segments; - - auto sop = tokenizer->encode("<|startofprev|>"); - auto sot = tokenizer->encode("<|startoftranscript|>"); - auto zero = tokenizer->encode("<|0.00|>"); - auto notimestamps = tokenizer->encode("<|notimestamps|>"); - const bool has_notimestamps = !notimestamps.empty() && - std::search(initial_tokens.begin(), initial_tokens.end(), - notimestamps.begin(), notimestamps.end()) != initial_tokens.end(); - auto sot_it = std::search(initial_tokens.begin(), initial_tokens.end(), sot.begin(), sot.end()); - const auto sot_begin = sot_it != initial_tokens.end() ? sot_it : initial_tokens.begin(); - - for (auto& audio_chunk : audio_chunks) { - if (handle->should_stop || completion_tokens >= options.max_tokens) break; - - std::vector chunk_audio = std::move(audio_chunk.audio); - const float audio_chunk_length_sec = static_cast(chunk_audio.size()) / static_cast(WHISPER_SAMPLE_RATE); - - std::vector tokens; - if (final_text.empty() || is_parakeet || is_moonshine) { - tokens = initial_tokens; - } else { - size_t word_count = 0, pos = final_text.size(); - while (pos > 0 && word_count < MAX_CONTEXT_WORDS) { - while (pos > 0 && std::isspace((unsigned char)final_text[pos - 1])) --pos; - while (pos > 0 && !std::isspace((unsigned char)final_text[pos - 1])) --pos; - ++word_count; - } - tokens = sop; - auto ctx = tokenizer->encode(final_text.substr(pos)); - tokens.insert(tokens.end(), ctx.begin(), ctx.end()); - tokens.insert(tokens.end(), sot_begin, initial_tokens.end()); - } - - if (is_whisper && !has_notimestamps && !zero.empty() && (tokens.empty() || tokens.back() != zero.back())) { - tokens.insert(tokens.end(), zero.begin(), zero.end()); - } - - if (!is_moonshine) { - if (is_parakeet) { - size_t waveform_samples = chunk_audio.size(); - apply_preemphasis(chunk_audio, 0.97f); - chunk_audio = ap.compute_spectrogram(chunk_audio, cfg); - normalize_parakeet_log_mel(chunk_audio, mel_bins); - size_t valid_frames = waveform_samples / cfg.hop_length; - if (valid_frames == 0) valid_frames = 1; - trim_mel_frames(chunk_audio, mel_bins, valid_frames); - } else { - std::vector mel = ap.compute_spectrogram(chunk_audio, cfg); - chunk_audio = normalize_whisper_mel(mel, mel_bins, is_whisper_v3); - } - } - - if (chunk_audio.empty()) { - CACTUS_LOG_DEBUG("transcribe", "Chunk audio features empty, skipping"); - continue; - } - - CACTUS_LOG_DEBUG("transcribe", "Chunk audio features size: " << chunk_audio.size()); - - size_t chunk_max_tokens = options.max_tokens - completion_tokens; - if (!is_parakeet) { - size_t max_allowed = tokens.size() < WHISPER_MAX_DECODER_POSITIONS ? - WHISPER_MAX_DECODER_POSITIONS - tokens.size() : 0; - if (chunk_max_tokens > max_allowed) chunk_max_tokens = max_allowed; - } - size_t max_tps_tokens = std::max(1, static_cast(audio_chunk_length_sec * max_tps)); - if (chunk_max_tokens > max_tps_tokens) chunk_max_tokens = max_tps_tokens; - - tokens.reserve(tokens.size() + chunk_max_tokens); - std::vector generated_tokens; - generated_tokens.reserve(chunk_max_tokens); - - float segment_start = audio_chunk.start_sec; - float segment_end = audio_chunk.end_sec; - float orig_offset = audio_chunk.start_sec; - size_t anchor_idx = 0; - std::string segment_text; - - for (size_t i = 0; i < chunk_max_tokens; ++i) { - if (handle->should_stop) break; - - float token_entropy = 0.0f; - float tok_time_start = 0.0f; - float tok_time_end = 0.0f; - uint32_t next_token = handle->model->decode_with_audio( - tokens, - chunk_audio, - options.temperature, - options.top_p, - options.top_k, - "", - &token_entropy, - options.min_p, - options.repetition_penalty, - is_parakeet_tdt ? &tok_time_start : nullptr, - is_parakeet_tdt ? &tok_time_end : nullptr - ); - - if (completion_tokens == 0) [[unlikely]] { - auto t_first = std::chrono::high_resolution_clock::now(); - time_to_first_token = std::chrono::duration_cast(t_first - start_time).count() / 1000.0; - } - - total_entropy_sum += token_entropy; - if (token_entropy > max_token_entropy_norm) max_token_entropy_norm = token_entropy; - - generated_tokens.emplace_back(next_token); - if (matches_stop_sequence(generated_tokens, stop_token_sequences)) { - break; - } - - std::string piece = tokenizer->decode({ next_token }); - if (is_terminal_transcription_piece(piece)) { - break; - } - tokens.emplace_back(next_token); - completion_tokens++; - - std::smatch timestamp_match; - if (is_parakeet_tdt) { - while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= tok_time_end) { - ++anchor_idx; - orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; - } - const bool new_word = !piece.empty() && piece[0] == ' '; - if (new_word && !segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - segment_text.clear(); - } - if (segment_text.empty()) segment_start = tok_time_start + orig_offset; - segment_text += piece; - segment_end = tok_time_end + orig_offset; - if (callback) callback(piece.c_str(), next_token, user_data); - continue; - } else if (std::regex_match(piece, timestamp_match, whisper_timestamp_re)) { - const float ts_sec = std::stof(timestamp_match[1].str()); - while (anchor_idx + 1 < audio_chunk.anchors.size() && audio_chunk.anchors[anchor_idx + 1].first <= ts_sec) { - ++anchor_idx; - orig_offset = audio_chunk.anchors[anchor_idx].second - audio_chunk.anchors[anchor_idx].first; - } - segment_end = std::min(ts_sec + orig_offset, audio_chunk.end_sec); - if (!segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - segment_text.clear(); - } - segment_start = segment_end; - continue; - } - - segment_text += piece; - if (callback) callback(piece.c_str(), next_token, user_data); - } - - if (!segment_text.empty()) { - final_text += segment_text; - segments.emplace_back( - segment_start, - segment_end, - segment_text[0] == ' ' - ? (segment_text.erase(0, 1), std::move(segment_text)) - : std::move(segment_text) - ); - } - - cactus_reset(model); - } - - float mean_entropy = completion_tokens > 0 ? total_entropy_sum / static_cast(completion_tokens) : 0.0f; - float confidence = 1.0f - mean_entropy; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - size_t prompt_tokens = initial_tokens.size(); - double prefill_tps = time_to_first_token > 0 ? (prompt_tokens * 1000.0) / time_to_first_token : 0.0; - double decode_time_ms = std::max(0.0, total_time_ms - time_to_first_token); - double decode_tps = (completion_tokens > 1 && decode_time_ms > 0.0) ? ((completion_tokens - 1) * 1000.0) / decode_time_ms : 0.0; - - if (!final_text.empty() && final_text[0] == ' ') final_text.erase(0, 1); - - if (!custom_vocabulary.empty()) { - apply_vocabulary_spelling_correction(final_text, custom_vocabulary); - for (auto& seg : segments) { - apply_vocabulary_spelling_correction(seg.text, custom_vocabulary); - } - } - - const bool cloud_handoff = !final_text.empty() && final_text.length() > 5 && - cloud_handoff_threshold > 0.0f && max_token_entropy_norm > cloud_handoff_threshold; - - std::string json = construct_response_json(final_text, {}, time_to_first_token, total_time_ms, prefill_tps, decode_tps, prompt_tokens, completion_tokens, confidence, cloud_handoff, "", segments); - - if (json.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(handle->model_name.c_str(), false, 0.0, 0.0, 0.0, 0, 0.0, "Response buffer too small"); - return -1; - } - - cactus::telemetry::recordTranscription(handle->model_name.c_str(), true, time_to_first_token, decode_tps, total_time_ms, static_cast(completion_tokens), get_ram_usage_mb(), ""); - - std::strcpy(response_buffer, json.c_str()); - - return static_cast(json.size()); - } - catch (const std::exception& e) { - CACTUS_LOG_ERROR("transcribe", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, e.what()); - return -1; - } - catch (...) { - CACTUS_LOG_ERROR("transcribe", "Unknown exception during transcription"); - handle_error_response("Unknown error in transcribe", response_buffer, buffer_size); - cactus::telemetry::recordTranscription(model ? static_cast(model)->model_name.c_str() : nullptr, false, 0.0, 0.0, 0.0, 0, 0.0, "Unknown error in transcribe"); - return -1; - } -} - -int cactus_detect_language( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (!model) { - std::string error_msg = last_error_message.empty() ? "Model not initialized." : last_error_message; - CACTUS_LOG_ERROR("detect_language", error_msg); - handle_error_response(error_msg, response_buffer, buffer_size); - return -1; - } - if (!response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR("detect_language", "Invalid parameters: response_buffer or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - CACTUS_LOG_ERROR("detect_language", "No audio input provided"); - handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); - return -1; - } - if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { - CACTUS_LOG_ERROR("detect_language", "Both audio_file_path and pcm_buffer provided"); - handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); - return -1; - } - if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { - CACTUS_LOG_ERROR("detect_language", "Invalid pcm_buffer_size: " << pcm_buffer_size); - handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); - return -1; - } - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - std::lock_guard lock(handle->model_mutex); - handle->should_stop = false; - - if (handle->model->get_config().model_type != cactus::engine::Config::ModelType::WHISPER) { - handle_error_response("Language detection currently requires a Whisper model", response_buffer, buffer_size); - return -1; - } - - InferenceOptions options = parse_inference_options_json(options_json ? options_json : ""); - - std::vector audio_buffer; - if (audio_file_path == nullptr) { - const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); - size_t num_samples = pcm_buffer_size / 2; - - std::vector waveform_fp32(num_samples); - for (size_t i = 0; i < num_samples; i++) { - waveform_fp32[i] = static_cast(pcm_samples[i]) / 32768.0f; - } - audio_buffer = resample_to_16k_fp32(waveform_fp32, WHISPER_SAMPLE_RATE); - } else { - AudioFP32 audio = load_wav(audio_file_path); - audio_buffer = resample_to_16k_fp32(audio.samples, audio.sample_rate); - } - - if (options.use_vad && handle->vad_model) { - auto* vad = static_cast(handle->vad_model.get()); - auto segments = vad->get_speech_timestamps(audio_buffer, {}); - - std::vector speech_audio; - for (const auto& segment : segments) { - speech_audio.insert( - speech_audio.end(), - audio_buffer.begin() + segment.start, - audio_buffer.begin() + std::min(segment.end, audio_buffer.size()) - ); - } - audio_buffer = std::move(speech_audio); - } - - if (audio_buffer.empty()) { - handle_error_response("No speech detected in audio input", response_buffer, buffer_size); - return -1; - } - - AudioProcessor ap; - auto cfg = get_whisper_spectrogram_config(); - const size_t mel_bins = std::max(1, static_cast(handle->model->get_config().num_mel_bins)); - init_whisper_mel_filters(ap, cfg, mel_bins); - std::vector mel = ap.compute_spectrogram(audio_buffer, cfg); - std::vector features = normalize_whisper_mel(mel, mel_bins, mel_bins > 80); - if (features.empty()) { - handle_error_response("Computed audio features are empty", response_buffer, buffer_size); - return -1; - } - - auto* tokenizer = handle->model->get_tokenizer(); - if (!tokenizer) { - handle_error_response("Tokenizer unavailable", response_buffer, buffer_size); - return -1; - } - - std::vector decode_tokens = tokenizer->encode("<|startoftranscript|>"); - if (decode_tokens.empty()) { - handle_error_response("Failed to encode Whisper start token", response_buffer, buffer_size); - return -1; - } - - handle->model->reset_cache(); - float entropy = 1.0f; - uint32_t token_id = 0; - std::string token_text; - std::string language; - - // Whisper may emit one or more control tokens before the language token. - // Decode a few initial tokens and select the first valid language token. - constexpr size_t kMaxLanguageProbeSteps = 4; - for (size_t step = 0; step < kMaxLanguageProbeSteps; ++step) { - float step_entropy = 1.0f; - const uint32_t step_token_id = handle->model->decode_with_audio( - decode_tokens, features, 0.0f, 0.0f, 0, "", &step_entropy, - 0.0f, 1.0f - ); - const std::string step_token_text = tokenizer->decode({step_token_id}); - const std::string step_language = extract_whisper_language_code(step_token_text); - - token_id = step_token_id; - token_text = step_token_text; - entropy = step_entropy; - - if (!step_language.empty()) { - language = step_language; - break; - } - - decode_tokens.push_back(step_token_id); - } - handle->model->reset_cache(); - - if (language.empty()) { - language = "unknown"; - } - - float confidence = 1.0f - entropy; - if (confidence < 0.0f) confidence = 0.0f; - if (confidence > 1.0f) confidence = 1.0f; - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"language\":\"" << escape_json_string(language) << "\","; - json << "\"language_token\":\"" << escape_json_string(token_text) << "\","; - json << "\"token_id\":" << token_id << ","; - json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; - json << "\"entropy\":" << std::fixed << std::setprecision(4) << entropy << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.size() >= buffer_size) { - handle_error_response("Response buffer too small", response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.size()); - } catch (const std::exception& e) { - CACTUS_LOG_ERROR("detect_language", "Exception: " << e.what()); - handle_error_response(e.what(), response_buffer, buffer_size); - return -1; - } catch (...) { - CACTUS_LOG_ERROR("detect_language", "Unknown exception during language detection"); - handle_error_response("Unknown error in detect_language", response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus/ffi/cactus_utils.h b/cactus/ffi/cactus_utils.h deleted file mode 100644 index aad98d83c..000000000 --- a/cactus/ffi/cactus_utils.h +++ /dev/null @@ -1,1869 +0,0 @@ -#ifndef CACTUS_UTILS_H -#define CACTUS_UTILS_H - -#include "../engine/engine.h" -#include "../models/model.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#elif defined(_WIN32) -#include -#include -#elif defined(__linux__) || defined(__ANDROID__) -#include -#endif - -inline size_t get_memory_footprint_bytes() { -#ifdef __APPLE__ - task_vm_info_data_t vm_info; - mach_msg_type_number_t count = TASK_VM_INFO_COUNT; - if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count) == KERN_SUCCESS) - return vm_info.phys_footprint; - -#elif defined(_WIN32) - PROCESS_MEMORY_COUNTERS_EX pmc; - if (GetProcessMemoryInfo(GetCurrentProcess(), (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc))) - return pmc.PrivateUsage; - -#elif defined(__linux__) || defined(__ANDROID__) - std::ifstream statm("/proc/self/statm"); - if (statm.is_open()) { - size_t size, resident; - statm >> size >> resident; - return resident * sysconf(_SC_PAGESIZE); - } -#endif - return 0; -} - -inline double get_ram_usage_mb() { - return get_memory_footprint_bytes() / (1024.0 * 1024.0); -} - -struct CactusModelHandle { - std::unique_ptr model; - std::unique_ptr vad_model; - std::atomic should_stop; - std::vector processed_tokens; - struct ProcessedImage { - std::string path; - long long last_modified_timestamp = 0; - - bool operator==(const ProcessedImage& other) const { - return path == other.path && last_modified_timestamp == other.last_modified_timestamp; - } - }; - - std::vector> processed_images; - std::vector user_audio_counts; - std::mutex model_mutex; - std::string model_name; - std::unique_ptr corpus_index; - std::string corpus_dir; - size_t corpus_embedding_dim = 0; - std::vector> tool_embeddings; - std::vector tool_texts; - - CactusModelHandle() : should_stop(false) {} -}; - -extern std::string last_error_message; - -bool matches_stop_sequence(const std::vector& generated_tokens, - const std::vector>& stop_sequences); - -std::string retrieve_rag_context(CactusModelHandle* handle, const std::string& query); - -namespace cactus { -namespace audio { - -static constexpr size_t WHISPER_TARGET_FRAMES = 3000; -static constexpr int WHISPER_SAMPLE_RATE = 16000; - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_whisper_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 400; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = true; - cfg.pad_mode = "reflect"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 1e-10f; - cfg.log_mel = "log10"; - cfg.reference = 1.0f; - cfg.min_value = 1e-10f; - cfg.remove_dc_offset = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_parakeet_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 512; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = true; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 5.960464477539063e-08f; // 2^-24 guard value used by HF Parakeet. - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 1e-10f; - cfg.remove_dc_offset = false; - cfg.hann_periodic = false; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_htk_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 321; - cfg.frame_length = 320; - cfg.fft_override = 1024; - cfg.hop_length = 160; - cfg.power = 1.0f; - cfg.center = false; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 0.001f; - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 0.001f; - cfg.remove_dc_offset = false; - cfg.hann_periodic = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_gemma4_audio_spectrogram_config( - const cactus::engine::Config& model_config) { - auto cfg = get_htk_spectrogram_config(); - cfg.fft_override = model_config.audio_fft_length; - cfg.mel_floor_additive = true; - return cfg; -} - -inline cactus::engine::AudioProcessor::SpectrogramConfig get_wespeaker_spectrogram_config() { - cactus::engine::AudioProcessor::SpectrogramConfig cfg{}; - cfg.n_fft = 512; - cfg.frame_length = 400; - cfg.hop_length = 160; - cfg.power = 2.0f; - cfg.center = false; - cfg.pad_mode = "constant"; - cfg.onesided = true; - cfg.dither = 0.0f; - cfg.mel_floor = 1.1754944e-38f; - cfg.log_mel = "log"; - cfg.reference = 1.0f; - cfg.min_value = 1.1754944e-38f; - cfg.remove_dc_offset = true; - cfg.preemphasis = 0.97f; - cfg.hann_periodic = false; - cfg.window_a0 = 0.54f; - return cfg; -} - -// Whisper v1/v2: 80 mel bins, HTK. Whisper v3: 128 mel bins, Slaney, 512-FFT, no DC removal. -inline void init_whisper_mel_filters(cactus::engine::AudioProcessor& ap, - cactus::engine::AudioProcessor::SpectrogramConfig& cfg, - size_t mel_bins) { - const size_t num_mel_filters = std::max(1, mel_bins); - const bool is_v3 = mel_bins > 80; - if (is_v3) { - cfg.fft_override = 512; - cfg.remove_dc_offset = false; - } - const size_t fft_len = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; - const size_t num_frequency_bins = fft_len / 2 + 1; - if (is_v3) { - ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, - WHISPER_SAMPLE_RATE, "slaney", "slaney"); - } else { - ap.init_mel_filters(num_frequency_bins, num_mel_filters, 0.0f, 8000.0f, - WHISPER_SAMPLE_RATE); - } -} - -// use_mel_floor_padding=true pads short audio with the normalized mel floor (required for v3). -inline std::vector normalize_whisper_mel(std::vector& mel, size_t n_mels, - bool use_mel_floor_padding = false) { - if (mel.empty() || n_mels == 0) return mel; - size_t n_frames = mel.size() / n_mels; - - float max_val = -std::numeric_limits::infinity(); - for (float v : mel) if (v > max_val) max_val = v; - - float min_allowed = max_val - 8.0f; - for (float& v : mel) { - if (v < min_allowed) v = min_allowed; - v = (v + 4.0f) * 0.25f; - } - - if (n_frames != WHISPER_TARGET_FRAMES) { - float pad_val = use_mel_floor_padding ? (min_allowed + 4.0f) * 0.25f : 0.0f; - std::vector fixed(n_mels * WHISPER_TARGET_FRAMES, pad_val); - size_t copy_frames = std::min(n_frames, WHISPER_TARGET_FRAMES); - for (size_t m = 0; m < n_mels; ++m) { - const float* src = &mel[m * n_frames]; - float* dst = &fixed[m * WHISPER_TARGET_FRAMES]; - std::copy(src, src + copy_frames, dst); - } - return fixed; - } - return std::move(mel); -} - -inline std::vector transpose_mel_to_frame_major(const std::vector& mel, - size_t num_mels, size_t num_frames) { - std::vector transposed(num_frames * num_mels); - for (size_t m = 0; m < num_mels; m++) { - for (size_t t = 0; t < num_frames; t++) { - transposed[t * num_mels + m] = mel[m * num_frames + t]; - } - } - return transposed; -} - -inline void apply_preemphasis(std::vector& waveform, float coefficient = 0.97f) { - if (waveform.size() < 2 || coefficient == 0.0f) { - return; - } - for (size_t i = waveform.size() - 1; i > 0; --i) { - waveform[i] -= coefficient * waveform[i - 1]; - } -} - -inline void normalize_parakeet_log_mel(std::vector& mel, size_t num_mels, float epsilon = 1e-5f) { - if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { - return; - } - const size_t num_frames = mel.size() / num_mels; - if (num_frames == 0) { - return; - } - - for (size_t m = 0; m < num_mels; ++m) { - const size_t base = m * num_frames; - float mean = 0.0f; - for (size_t t = 0; t < num_frames; ++t) { - mean += mel[base + t]; - } - mean /= static_cast(num_frames); - - float variance = 0.0f; - for (size_t t = 0; t < num_frames; ++t) { - const float d = mel[base + t] - mean; - variance += d * d; - } - const float denom = static_cast(std::max(1, num_frames - 1)); - const float inv_std = 1.0f / std::sqrt((variance / denom) + epsilon); - for (size_t t = 0; t < num_frames; ++t) { - mel[base + t] = (mel[base + t] - mean) * inv_std; - } - } -} - -inline void trim_mel_frames(std::vector& mel, size_t num_mels, size_t valid_frames) { - if (mel.empty() || num_mels == 0 || (mel.size() % num_mels) != 0) { - return; - } - size_t total_frames = mel.size() / num_mels; - if (valid_frames == 0 || valid_frames >= total_frames) { - return; - } - std::vector trimmed(num_mels * valid_frames); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &mel[m * total_frames]; - float* dst = &trimmed[m * valid_frames]; - std::copy(src, src + valid_frames, dst); - } - mel.swap(trimmed); -} - -struct AudioPreprocessResult { - std::vector features; - size_t num_frames = 0; - size_t num_soft_tokens = 0; -}; - -inline AudioPreprocessResult preprocess_audio_for_gemma4( - std::vector audio_samples, - const cactus::engine::Config& model_config -) { - AudioPreprocessResult result; - if (audio_samples.empty()) return result; - - size_t pad_amt = 320 - (audio_samples.size() % 320); - if (pad_amt < 320) - audio_samples.resize(audio_samples.size() + pad_amt, 0.0f); - - size_t mel_bins = model_config.audio_input_feat_size; - auto cfg = get_gemma4_audio_spectrogram_config(model_config); - - size_t semicausal_pad = cfg.frame_length / 2; - audio_samples.insert(audio_samples.begin(), semicausal_pad, 0.0f); - - cactus::engine::AudioProcessor ap; - size_t fft_for_mel = cfg.fft_override > 0 ? cfg.fft_override : cfg.n_fft; - ap.init_mel_filters(fft_for_mel / 2 + 1, mel_bins, 0.0f, 8000.0f, 16000, - nullptr, "htk"); - std::vector mel = ap.compute_spectrogram(audio_samples, cfg); - - result.num_frames = mel.size() / mel_bins; - result.features = transpose_mel_to_frame_major(mel, mel_bins, result.num_frames); - - size_t after_stage1 = (result.num_frames + 1) / 2; - result.num_soft_tokens = (after_stage1 + 1) / 2; - - return result; -} - -inline std::vector pcm_buffer_to_float_samples( - const uint8_t* pcm_buffer, size_t pcm_buffer_size -) { - const int16_t* pcm_samples = reinterpret_cast(pcm_buffer); - size_t num_samples = pcm_buffer_size / 2; - std::vector waveform_fp32(num_samples); - constexpr float inv_32768 = 1.0f / 32768.0f; - for (size_t i = 0; i < num_samples; i++) - waveform_fp32[i] = static_cast(pcm_samples[i]) * inv_32768; - return waveform_fp32; -} - -} // namespace audio -} // namespace cactus - -namespace cactus { -namespace ffi { - -inline bool env_flag_enabled(const char* key) { - const char* value = std::getenv(key); - return value && value[0] != '\0' && !(value[0] == '0' && value[1] == '\0'); -} - -inline std::string generateUUID() { -#ifdef __APPLE__ - uuid_t uuid; - uuid_generate_random(uuid); - char uuid_str[37]; - uuid_unparse_lower(uuid, uuid_str); - return std::string(uuid_str); -#else - static std::random_device rd; - static std::mt19937 gen(rd()); - static std::uniform_int_distribution<> dis(0, 15); - static std::uniform_int_distribution<> dis2(8, 11); - - std::stringstream ss; - ss << std::hex; - for (int i = 0; i < 8; i++) ss << dis(gen); - ss << "-"; - for (int i = 0; i < 4; i++) ss << dis(gen); - ss << "-4"; - for (int i = 0; i < 3; i++) ss << dis(gen); - ss << "-"; - ss << dis2(gen); - for (int i = 0; i < 3; i++) ss << dis(gen); - ss << "-"; - for (int i = 0; i < 12; i++) ss << dis(gen); - return ss.str(); -#endif -} - -struct ToolFunction { - std::string name; - std::string description; - std::unordered_map parameters; -}; - -struct InferenceOptions { - float temperature = 0.0f; - float top_p = 0.0f; - float min_p = 0.15f; - float repetition_penalty = 1.1f; - float confidence_threshold = -1.0f; - size_t top_k = 0; - size_t max_tokens = 100; - size_t tool_rag_top_k = 2; - size_t cloud_timeout_ms = 15000; - std::vector stop_sequences; - bool force_tools = false; - bool include_stop_sequences = false; - bool use_vad = true; - bool telemetry_enabled = true; - bool auto_handoff = true; - bool handoff_with_images = true; - bool enable_thinking_if_supported = false; -}; - -} // namespace ffi -} // namespace cactus - -std::vector select_relevant_tools( - CactusModelHandle* handle, - const std::string& query, - const std::vector& all_tools, - size_t top_k); - -#include "gemma_tools.h" - -namespace cactus { -namespace ffi { - -inline std::string escape_json_string(const std::string& s) { - std::ostringstream o; - for (char c : s) { - if (c == '"') o << "\\\""; - else if (c == '\n') o << "\\n"; - else if (c == '\r') o << "\\r"; - else if (c == '\t') o << "\\t"; - else if (c == '\\') o << "\\\\"; - else o << c; - } - return o.str(); -} - - -inline std::string trim_string(const std::string& s) { - size_t start = 0; - while (start < s.size() && std::isspace(static_cast(s[start]))) ++start; - size_t end = s.size(); - while (end > start && std::isspace(static_cast(s[end - 1]))) --end; - return s.substr(start, end - start); -} - -inline size_t find_matching_delimiter(const std::string& s, size_t pos, char open, char close) { - int depth = 1; - pos++; - while (pos < s.length() && depth > 0) { - if (s[pos] == open) depth++; - else if (s[pos] == close) depth--; - else if (s[pos] == '"') { - pos++; - while (pos < s.length() && s[pos] != '"') { - if (s[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - return pos; -} - -inline std::string env_or_default(const char* key, const char* fallback) { - const char* v = std::getenv(key); - if (v && v[0] != '\0') return std::string(v); - return std::string(fallback); -} - -inline std::string json_string_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return {}; - - size_t i = pos + pattern.size(); - while (i < json.size() && std::isspace(static_cast(json[i]))) i++; - if (i >= json.size() || json[i] != '"') return {}; - ++i; - - std::string out; - out.reserve(128); - while (i < json.size()) { - char c = json[i++]; - if (c == '"') return out; - if (c == '\\' && i < json.size()) { - char e = json[i++]; - switch (e) { - case '"': out.push_back('"'); break; - case '\\': out.push_back('\\'); break; - case '/': out.push_back('/'); break; - case 'b': out.push_back('\b'); break; - case 'f': out.push_back('\f'); break; - case 'n': out.push_back('\n'); break; - case 'r': out.push_back('\r'); break; - case 't': out.push_back('\t'); break; - default: out.push_back(e); break; - } - continue; - } - out.push_back(c); - } - return {}; -} - -inline std::string json_array_field(const std::string& json, const std::string& key) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return "[]"; - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - if (start >= json.size() || json[start] != '[') return "[]"; - - int depth = 1; - size_t end = start + 1; - while (end < json.size() && depth > 0) { - if (json[end] == '[') depth++; - else if (json[end] == ']') depth--; - end++; - } - return json.substr(start, end - start); -} - -inline std::vector split_json_array(const std::string& array_json) { - std::vector out; - if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; - - size_t i = 1; - while (i + 1 < array_json.size()) { - while (i + 1 < array_json.size() && - (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) i++; - if (i + 1 >= array_json.size() || array_json[i] != '{') break; - - size_t start = i; - int depth = 0; - bool in_str = false; - bool esc = false; - for (; i < array_json.size(); ++i) { - char c = array_json[i]; - if (in_str) { - if (esc) esc = false; - else if (c == '\\') esc = true; - else if (c == '"') in_str = false; - continue; - } - if (c == '"') { in_str = true; continue; } - if (c == '{') depth++; - if (c == '}') { - depth--; - if (depth == 0) { - out.push_back(array_json.substr(start, i - start + 1)); - i++; - break; - } - } - } - } - return out; -} - -inline std::string serialize_tools_json(const std::vector& tools) { - if (tools.empty()) return ""; - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < tools.size(); ++i) { - if (i > 0) oss << ","; - oss << "{\"type\":\"function\",\"function\":{"; - oss << "\"name\":\"" << escape_json_string(tools[i].name) << "\","; - oss << "\"description\":\"" << escape_json_string(tools[i].description) << "\""; - auto it = tools[i].parameters.find("schema"); - if (it != tools[i].parameters.end()) { - oss << ",\"parameters\":" << it->second; - } - oss << "}}"; - } - oss << "]"; - return oss.str(); -} - -namespace json_sorted { - -inline void skip_ws(const std::string& s, size_t& p) { - while (p < s.size() && std::isspace(static_cast(s[p]))) p++; -} - -inline std::string parse_string(const std::string& s, size_t& p) { - std::string r = "\""; - p++; - while (p < s.size()) { - if (s[p] == '\\') { - r += s[p++]; - if (p < s.size()) r += s[p++]; - } else if (s[p] == '"') { - r += '"'; - p++; - return r; - } else { - r += s[p++]; - } - } - return r; -} - -inline std::string parse_value(const std::string& s, size_t& p); - -inline std::string parse_object(const std::string& s, size_t& p) { - p++; - std::map entries; - skip_ws(s, p); - while (p < s.size() && s[p] != '}') { - if (s[p] == ',') { p++; skip_ws(s, p); continue; } - std::string key = parse_string(s, p); - skip_ws(s, p); - if (p < s.size() && s[p] == ':') p++; - skip_ws(s, p); - std::string val = parse_value(s, p); - entries[key] = val; - skip_ws(s, p); - } - if (p < s.size()) p++; - std::string r = "{"; - bool first = true; - for (const auto& kv : entries) { - if (!first) r += ", "; - r += kv.first + ": " + kv.second; - first = false; - } - r += "}"; - return r; -} - -inline std::string parse_array(const std::string& s, size_t& p) { - p++; - std::vector items; - skip_ws(s, p); - while (p < s.size() && s[p] != ']') { - if (s[p] == ',') { p++; skip_ws(s, p); continue; } - items.push_back(parse_value(s, p)); - skip_ws(s, p); - } - if (p < s.size()) p++; - std::string r = "["; - for (size_t i = 0; i < items.size(); i++) { - if (i > 0) r += ", "; - r += items[i]; - } - r += "]"; - return r; -} - -inline std::string parse_value(const std::string& s, size_t& p) { - skip_ws(s, p); - if (p >= s.size()) return ""; - if (s[p] == '"') return parse_string(s, p); - if (s[p] == '{') return parse_object(s, p); - if (s[p] == '[') return parse_array(s, p); - size_t start = p; - while (p < s.size() && s[p] != ',' && s[p] != '}' && s[p] != ']' && !std::isspace(static_cast(s[p]))) p++; - return s.substr(start, p - start); -} - -inline std::string reformat(const std::string& json) { - size_t p = 0; - return parse_value(json, p); -} - -} // namespace json_sorted - -inline std::string serialize_tools_for_template(const std::vector& tools) { - if (tools.empty()) return ""; - std::string result; - for (const auto& tool : tools) { - std::map func_fields; - func_fields["\"description\""] = "\"" + escape_json_string(tool.description) + "\""; - func_fields["\"name\""] = "\"" + escape_json_string(tool.name) + "\""; - auto it = tool.parameters.find("schema"); - if (it != tool.parameters.end()) { - func_fields["\"parameters\""] = json_sorted::reformat(it->second); - } - std::string func_json = "{"; - bool first = true; - for (const auto& kv : func_fields) { - if (!first) func_json += ", "; - func_json += kv.first + ": " + kv.second; - first = false; - } - func_json += "}"; - result += "\n{\"function\": " + func_json + ", \"type\": \"function\"}"; - } - return result; -} - -inline void handle_error_response(const std::string& error_message, char* response_buffer, size_t buffer_size) { - std::ostringstream json; - json << "{"; - json << "\"success\":false,"; - json << "\"error\":\"" << escape_json_string(error_message) << "\","; - json << "\"cloud_handoff\":false,"; - json << "\"response\":null,"; - json << "\"function_calls\":[],"; - json << "\"confidence\":0.0,"; - json << "\"time_to_first_token_ms\":0.0,"; - json << "\"total_time_ms\":0.0,"; - json << "\"prefill_tps\":0.0,"; - json << "\"decode_tps\":0.0,"; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; - json << "\"prefill_tokens\":0,"; - json << "\"decode_tokens\":0,"; - json << "\"total_tokens\":0"; - json << "}"; - std::string error_json = json.str(); - if (response_buffer && error_json.length() < buffer_size) { - std::strcpy(response_buffer, error_json.c_str()); - } -} - -inline std::vector parse_messages_json(const std::string& json, - std::vector& out_image_paths, - std::vector* out_audio_paths = nullptr) { - std::vector messages; - out_image_paths.clear(); - if (out_audio_paths) out_audio_paths->clear(); - - size_t pos = json.find('['); - if (pos == std::string::npos) { - throw std::runtime_error("Invalid JSON: expected array"); - } - - pos = json.find('{', pos); - while (pos != std::string::npos) { - cactus::engine::ChatMessage msg; - - size_t obj_start = pos; - int brace_count = 1; - size_t obj_end = obj_start + 1; - while (obj_end < json.length() && brace_count > 0) { - if (json[obj_end] == '{') brace_count++; - else if (json[obj_end] == '}') brace_count--; - obj_end++; - } - - size_t role_pos = json.find("\"role\"", pos); - if (role_pos == std::string::npos || role_pos >= obj_end) break; - - size_t role_start = json.find('"', role_pos + 6) + 1; - size_t role_end = json.find('"', role_start); - msg.role = json.substr(role_start, role_end - role_start); - - size_t content_pos = json.find("\"content\"", role_end); - if (content_pos != std::string::npos && content_pos < obj_end) { - size_t content_start = json.find('"', content_pos + 9) + 1; - size_t content_end = content_start; - - while (content_end < json.length()) { - content_end = json.find('"', content_end); - if (content_end == std::string::npos) break; - if (json[content_end - 1] != '\\') break; - content_end++; - } - - msg.content = json.substr(content_start, content_end - content_start); - - size_t escape_pos = 0; - while ((escape_pos = msg.content.find("\\n", escape_pos)) != std::string::npos) { - msg.content.replace(escape_pos, 2, "\n"); - escape_pos += 1; - } - escape_pos = 0; - while ((escape_pos = msg.content.find("\\\"", escape_pos)) != std::string::npos) { - msg.content.replace(escape_pos, 2, "\""); - escape_pos += 1; - } - } - - auto parse_path_array = [&](const char* key, std::vector& dest, - std::vector* out_paths) { - size_t key_pos = json.find(key, pos); - if (key_pos == std::string::npos || key_pos >= obj_end) return; - size_t array_start = json.find('[', key_pos); - if (array_start == std::string::npos || array_start >= obj_end) return; - size_t array_end = json.find(']', array_start); - if (array_end == std::string::npos || array_end >= obj_end) return; - size_t cur = array_start; - while (true) { - cur = json.find('"', cur + 1); - if (cur == std::string::npos || cur >= array_end) break; - size_t str_start = cur + 1; - size_t str_end = json.find('"', str_start); - if (str_end == std::string::npos || str_end > array_end) break; - std::string path = std::filesystem::absolute( - std::filesystem::path(json.substr(str_start, str_end - str_start))).string(); - dest.push_back(path); - if (out_paths) out_paths->push_back(path); - cur = str_end; - } - }; - - parse_path_array("\"images\"", msg.images, &out_image_paths); - parse_path_array("\"audio\"", msg.audio, out_audio_paths); - - if (msg.role == "tool") { - size_t name_pos = json.find("\"name\"", obj_start); - if (name_pos != std::string::npos && name_pos < obj_end) { - size_t name_quote = json.find('"', name_pos + 6); - if (name_quote != std::string::npos && name_quote < obj_end) { - size_t name_start = name_quote + 1; - size_t name_end = json.find('"', name_start); - if (name_end != std::string::npos && name_end < obj_end) { - msg.name = json.substr(name_start, name_end - name_start); - } - } - } - } - - size_t tool_calls_pos = json.find("\"tool_calls\"", obj_start); - if (tool_calls_pos != std::string::npos && tool_calls_pos < obj_end) { - size_t tool_calls_arr_start = json.find('[', tool_calls_pos); - if (tool_calls_arr_start != std::string::npos && tool_calls_arr_start < obj_end) { - size_t tool_calls_arr_end = find_matching_delimiter(json, tool_calls_arr_start, '[', ']'); - - size_t search_pos = tool_calls_arr_start; - while (true) { - size_t func_pos = json.find("\"function\"", search_pos); - if (func_pos == std::string::npos || func_pos >= tool_calls_arr_end) break; - - size_t func_obj_start = json.find('{', func_pos + 10); - if (func_obj_start == std::string::npos || func_obj_start >= tool_calls_arr_end) break; - - size_t func_obj_end = find_matching_delimiter(json, func_obj_start, '{', '}'); - - cactus::engine::ToolCallInfo tool_call; - - size_t fn_name_pos = json.find("\"name\"", func_obj_start); - if (fn_name_pos != std::string::npos && fn_name_pos < func_obj_end) { - size_t fn_name_quote = json.find('"', fn_name_pos + 6); - if (fn_name_quote != std::string::npos && fn_name_quote < func_obj_end) { - size_t fn_name_start = fn_name_quote + 1; - size_t fn_name_end = json.find('"', fn_name_start); - if (fn_name_end != std::string::npos && fn_name_end < func_obj_end) { - tool_call.name = json.substr(fn_name_start, fn_name_end - fn_name_start); - } - } - } - - size_t args_pos = json.find("\"arguments\"", func_obj_start); - if (args_pos != std::string::npos && args_pos < func_obj_end) { - size_t colon_pos = json.find(':', args_pos + 11); - if (colon_pos != std::string::npos && colon_pos < func_obj_end) { - size_t args_start = colon_pos + 1; - while (args_start < json.length() && std::isspace(static_cast(json[args_start]))) args_start++; - - if (args_start < func_obj_end && json[args_start] == '{') { - size_t args_end = find_matching_delimiter(json, args_start, '{', '}'); - tool_call.arguments = json.substr(args_start, args_end - args_start); - } else if (args_start < func_obj_end && json[args_start] == '"') { - size_t str_start = args_start + 1; - size_t str_end = str_start; - while (str_end < json.length() && json[str_end] != '"') { - if (json[str_end] == '\\') str_end++; - str_end++; - } - tool_call.arguments = json.substr(str_start, str_end - str_start); - } - } - } - - if (!tool_call.name.empty()) { - msg.tool_calls.push_back(tool_call); - } - search_pos = func_obj_end; - } - } - } - - messages.push_back(msg); - - pos = json.find('{', obj_end); - } - - return messages; -} - -inline std::vector parse_tools_json(const std::string& json) { - std::vector tools; - - if (json.empty()) return tools; - - size_t pos = json.find('['); - if (pos == std::string::npos) return tools; - - pos = json.find("\"function\"", pos); - while (pos != std::string::npos) { - ToolFunction tool; - size_t next_search = pos + 1; - - size_t name_pos = json.find("\"name\"", pos); - if (name_pos != std::string::npos) { - size_t name_start = json.find('"', name_pos + 6) + 1; - size_t name_end = json.find('"', name_start); - tool.name = json.substr(name_start, name_end - name_start); - } - - size_t desc_pos = json.find("\"description\"", pos); - if (desc_pos != std::string::npos) { - size_t desc_start = json.find('"', desc_pos + 13) + 1; - size_t desc_end = json.find('"', desc_start); - tool.description = json.substr(desc_start, desc_end - desc_start); - } - - size_t params_pos = json.find("\"parameters\"", pos); - if (params_pos != std::string::npos) { - size_t params_start = json.find('{', params_pos); - if (params_start != std::string::npos) { - int brace_count = 1; - size_t params_end = params_start + 1; - while (params_end < json.length() && brace_count > 0) { - if (json[params_end] == '{') brace_count++; - else if (json[params_end] == '}') brace_count--; - params_end++; - } - tool.parameters["schema"] = json.substr(params_start, params_end - params_start); - next_search = params_end; - } - } - - if (!tool.name.empty()) { - tools.push_back(tool); - } - - pos = json.find("\"function\"", next_search); - } - - return tools; -} - -inline bool try_parse_json_float(const std::string& json, const std::string& key, float& out_value) { - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return false; - - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - - size_t end = start; - while (end < json.size() && std::string(",}] \t\n\r").find(json[end]) == std::string::npos) ++end; - - try { - out_value = std::stof(json.substr(start, end - start)); - return true; - } catch (...) { - return false; - } -} - -inline std::vector parse_json_string_array_field(const std::string& json, const std::string& key) { - std::vector out; - std::string pattern = "\"" + key + "\":"; - size_t pos = json.find(pattern); - if (pos == std::string::npos) return out; - - size_t start = pos + pattern.size(); - while (start < json.size() && std::isspace(static_cast(json[start]))) ++start; - if (start >= json.size() || json[start] != '[') return out; - - int depth = 1; - bool in_string = false; - bool escaped = false; - size_t end = start + 1; - - while (end < json.size() && depth > 0) { - char c = json[end]; - if (in_string) { - if (escaped) escaped = false; - else if (c == '\\') escaped = true; - else if (c == '"') in_string = false; - } else { - if (c == '"') in_string = true; - else if (c == '[') depth++; - else if (c == ']') depth--; - } - ++end; - } - - if (depth != 0) return out; - const std::string array_json = json.substr(start, end - start); - if (array_json.size() < 2 || array_json.front() != '[' || array_json.back() != ']') return out; - - size_t i = 1; - while (i + 1 < array_json.size()) { - while (i + 1 < array_json.size() && - (std::isspace(static_cast(array_json[i])) || array_json[i] == ',')) { - ++i; - } - if (i + 1 >= array_json.size() || array_json[i] == ']') break; - if (array_json[i] != '"') break; - - ++i; - std::string value; - bool escaped = false; - while (i < array_json.size()) { - char c = array_json[i++]; - if (escaped) { - switch (c) { - case '"': value.push_back('"'); break; - case '\\': value.push_back('\\'); break; - case '/': value.push_back('/'); break; - case 'b': value.push_back('\b'); break; - case 'f': value.push_back('\f'); break; - case 'n': value.push_back('\n'); break; - case 'r': value.push_back('\r'); break; - case 't': value.push_back('\t'); break; - default: value.push_back(c); break; - } - escaped = false; - continue; - } - if (c == '\\') { - escaped = true; - continue; - } - if (c == '"') { - out.push_back(value); - break; - } - value.push_back(c); - } - } - - return out; -} - -inline void parse_custom_vocabulary_options(const std::string& json, - std::vector& custom_vocabulary, - float& vocabulary_boost) { - custom_vocabulary.clear(); - vocabulary_boost = 5.0f; - if (json.empty()) return; - - float parsed_boost = vocabulary_boost; - if (try_parse_json_float(json, "vocabulary_boost", parsed_boost)) { - vocabulary_boost = std::clamp(parsed_boost, 0.0f, 20.0f); - } - - custom_vocabulary = parse_json_string_array_field(json, "custom_vocabulary"); -} - -inline std::unordered_map build_token_bias_map(const std::vector>& tokenized_entries, - float vocabulary_boost) { - std::unordered_map vocab_bias; - const float clamped_boost = std::clamp(vocabulary_boost, 0.0f, 20.0f); - if (clamped_boost == 0.0f) return vocab_bias; - - for (const auto& token_ids : tokenized_entries) { - for (uint32_t token_id : token_ids) { - float& entry = vocab_bias[token_id]; - if (entry < clamped_boost) { - entry = clamped_boost; - } - } - } - - return vocab_bias; -} - -inline std::unordered_map build_custom_vocabulary_bias(cactus::engine::Tokenizer* tokenizer, - const std::vector& custom_vocabulary, - float vocabulary_boost) { - if (!tokenizer || custom_vocabulary.empty()) return {}; - std::vector> tokenized_entries; - tokenized_entries.reserve(custom_vocabulary.size()); - - for (const auto& word : custom_vocabulary) { - if (word.empty()) continue; - tokenized_entries.push_back(tokenizer->encode(word)); - } - - return build_token_bias_map(tokenized_entries, vocabulary_boost); -} - -inline void apply_custom_vocabulary_options(cactus::engine::Model* model, const std::string& json) { - if (!model) return; - - std::vector custom_vocabulary; - float vocabulary_boost = 5.0f; - parse_custom_vocabulary_options(json, custom_vocabulary, vocabulary_boost); - model->set_vocab_bias(build_custom_vocabulary_bias(model->get_tokenizer(), custom_vocabulary, vocabulary_boost)); -} - -inline size_t levenshtein_ci(const std::string& a, const std::string& b) { - const size_t m = a.size(), n = b.size(); - std::vector prev(n + 1), curr(n + 1); - for (size_t j = 0; j <= n; ++j) prev[j] = j; - for (size_t i = 1; i <= m; ++i) { - curr[0] = i; - for (size_t j = 1; j <= n; ++j) { - const bool match = std::tolower(static_cast(a[i - 1])) == - std::tolower(static_cast(b[j - 1])); - curr[j] = std::min({prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + (match ? 0 : 1)}); - } - std::swap(prev, curr); - } - return prev[n]; -} - -inline std::string collapse_spaces(const std::string& s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - if (c != ' ') out += c; - } - return out; -} - -inline void apply_vocabulary_spelling_correction( - std::string& text, - const std::vector& custom_vocabulary) -{ - if (custom_vocabulary.empty() || text.empty()) return; - - struct VocabEntry { - const std::string* original; - std::string collapsed; - }; - std::vector vocab_entries; - vocab_entries.reserve(custom_vocabulary.size()); - for (const auto& v : custom_vocabulary) { - vocab_entries.push_back({&v, collapse_spaces(v)}); - } - - struct Token { std::string text; bool is_word; }; - std::vector tokens; - size_t pos = 0; - while (pos < text.size()) { - if (std::isalnum(static_cast(text[pos])) || - text[pos] == '\'' || text[pos] == '-') { - size_t start = pos; - while (pos < text.size() && (std::isalnum(static_cast(text[pos])) || - text[pos] == '\'' || text[pos] == '-')) { - ++pos; - } - tokens.push_back({text.substr(start, pos - start), true}); - } else { - size_t start = pos; - while (pos < text.size() && !std::isalnum(static_cast(text[pos])) && - text[pos] != '\'' && text[pos] != '-') { - ++pos; - } - tokens.push_back({text.substr(start, pos - start), false}); - } - } - - std::vector word_indices; - for (size_t i = 0; i < tokens.size(); ++i) { - if (tokens[i].is_word) word_indices.push_back(i); - } - - std::vector consumed(tokens.size(), false); - - auto strip_suffix = [](const std::string& word) -> std::pair { - if (word.size() >= 3 && word.substr(word.size() - 2) == "'s") { - return {word.substr(0, word.size() - 2), "'s"}; - } - if (word.size() >= 3 && word.substr(word.size() - 2) == "'t") { - return {word.substr(0, word.size() - 2), "'t"}; - } - if (word.size() >= 4 && word.back() == 's' && - word[word.size() - 2] != 's' && // avoid stripping from "boss", "class" - std::isalpha(static_cast(word[word.size() - 2]))) { - return {word.substr(0, word.size() - 1), "s"}; - } - return {word, ""}; - }; - - size_t wi = 0; - while (wi < word_indices.size()) { - size_t best_dist = std::numeric_limits::max(); - const std::string* best_match = nullptr; - size_t best_window = 0; - size_t best_first_token = 0; - size_t best_last_token = 0; - std::string best_suffix; - - for (size_t window = std::min(3, word_indices.size() - wi); window >= 1; --window) { - std::string window_collapsed; - const size_t first_tok = word_indices[wi]; - const size_t last_tok = word_indices[wi + window - 1]; - for (size_t w = 0; w < window; ++w) { - window_collapsed += tokens[word_indices[wi + w]].text; - } - - if (window == 1 && window_collapsed.size() < 3) break; - - auto [stem, suffix] = strip_suffix(window_collapsed); - const std::string* candidates[] = {&window_collapsed, &stem}; - const std::string suffixes[] = {"", suffix}; - const size_t num_candidates = suffix.empty() ? 1 : 2; - - for (size_t ci = 0; ci < num_candidates; ++ci) { - const std::string& candidate = *candidates[ci]; - if (candidate.empty()) continue; - - for (const auto& entry : vocab_entries) { - const size_t wlen = candidate.size(); - const size_t vlen = entry.collapsed.size(); - - const size_t len_diff = wlen > vlen ? wlen - vlen : vlen - wlen; - const size_t max_dist = std::max(1, std::min(wlen, vlen) / 3); - if (len_diff > max_dist) continue; - - const size_t dist = levenshtein_ci(candidate, entry.collapsed); - - // For single-edit corrections, require first char match to prevent - // false positives like "vortex" → "Cortex". - if (dist == 1 && window == 1) { - const bool first_char_match = - std::tolower(static_cast(candidate[0])) == - std::tolower(static_cast(entry.collapsed[0])); - if (!first_char_match) continue; - } - - if (dist <= max_dist && dist < best_dist) { - best_dist = dist; - best_match = entry.original; - best_window = window; - best_first_token = first_tok; - best_last_token = last_tok; - best_suffix = suffixes[ci]; - } - } - } - - if (best_dist == 0) break; - } - - // Allow dist==0 for multi-word merges where word boundaries changed. - const bool should_replace = best_match && - best_dist != std::numeric_limits::max() && - (best_dist > 0 || best_window > 1); - - if (should_replace) { - tokens[best_first_token].text = *best_match + best_suffix; - for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { - consumed[t] = true; - } - for (size_t t = best_first_token + 1; t <= best_last_token; ++t) { - if (t > 0) consumed[t - 1] = consumed[t - 1] || !tokens[t - 1].is_word; - } - wi += best_window; - } else { - ++wi; - } - } - - std::string result; - result.reserve(text.size()); - for (size_t i = 0; i < tokens.size(); ++i) { - if (!consumed[i]) { - result += tokens[i].text; - } - } - - text = std::move(result); -} - -inline InferenceOptions parse_inference_options_json(const std::string& json) { - InferenceOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"temperature\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.temperature = std::stof(json.substr(pos)); - } - - pos = json.find("\"top_p\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.top_p = std::stof(json.substr(pos)); - } - - float parsed_min_p = options.min_p; - if (try_parse_json_float(json, "min_p", parsed_min_p)) { - options.min_p = std::clamp(parsed_min_p, 0.0f, 1.0f); - } - - float parsed_rep_penalty = options.repetition_penalty; - if (try_parse_json_float(json, "repetition_penalty", parsed_rep_penalty)) { - if (std::isfinite(parsed_rep_penalty) && parsed_rep_penalty > 0.0f) { - options.repetition_penalty = parsed_rep_penalty; - } - } - - pos = json.find("\"top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"max_tokens\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.max_tokens = std::stoul(json.substr(pos)); - } - - pos = json.find("\"force_tools\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.force_tools = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"tool_rag_top_k\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.tool_rag_top_k = std::stoul(json.substr(pos)); - } - - pos = json.find("\"confidence_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.confidence_threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"include_stop_sequences\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.include_stop_sequences = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"use_vad\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.use_vad = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"telemetry_enabled\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.telemetry_enabled = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"auto_handoff\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.auto_handoff = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"cloud_timeout_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - options.cloud_timeout_ms = std::stoul(json.substr(pos)); - } - - pos = json.find("\"handoff_with_images\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.handoff_with_images = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"enable_thinking_if_supported\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(static_cast(json[pos]))) pos++; - options.enable_thinking_if_supported = (json.substr(pos, 4) == "true"); - } - - pos = json.find("\"stop_sequences\""); - if (pos != std::string::npos) { - pos = json.find('[', pos); - if (pos != std::string::npos) { - size_t end_pos = json.find(']', pos); - size_t seq_pos = json.find('"', pos); - - while (seq_pos != std::string::npos && seq_pos < end_pos) { - size_t seq_start = seq_pos + 1; - size_t seq_end = json.find('"', seq_start); - if (seq_end != std::string::npos) { - options.stop_sequences.push_back(json.substr(seq_start, seq_end - seq_start)); - } - seq_pos = json.find('"', seq_end + 1); - } - } - } - - return options; -} - -static inline std::string trim_lfm2_slice(const std::string& value, size_t begin, size_t end) { - return trim_string(value.substr(begin, end - begin)); -} - -static inline void append_lfm2_call(const std::string& entry, - std::vector& function_calls) { - if (entry.empty()) return; - - std::string trimmed_entry = trim_lfm2_slice(entry, 0, entry.size()); - if (trimmed_entry.empty()) return; - - size_t paren_pos = trimmed_entry.find('('); - if (paren_pos == std::string::npos) return; - - std::string func_name = trim_lfm2_slice(trimmed_entry, 0, paren_pos); - std::string args_str = trim_lfm2_slice(trimmed_entry, paren_pos + 1, trimmed_entry.size()); - - while (!args_str.empty() && (args_str.back() == ')' || args_str.back() == ']')) { - args_str.pop_back(); - } - args_str = trim_lfm2_slice(args_str, 0, args_str.size()); - - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":{"; - - size_t arg_pos = 0; - bool first_arg = true; - while (arg_pos < args_str.length()) { - while (arg_pos < args_str.length() && std::isspace(static_cast(args_str[arg_pos]))) { - arg_pos++; - } - - size_t eq_pos = args_str.find('=', arg_pos); - if (eq_pos == std::string::npos) break; - - std::string arg_name = args_str.substr(arg_pos, eq_pos - arg_pos); - - size_t val_start = eq_pos + 1; - size_t val_end = val_start; - bool quoted = false; - - if (val_start < args_str.length() && args_str[val_start] == '"') { - quoted = true; - val_start++; - val_end = args_str.find('"', val_start); - if (val_end == std::string::npos) break; - } else { - int depth = 0; - for (val_end = val_start; val_end < args_str.length(); val_end++) { - char c = args_str[val_end]; - if (c == '[' || c == '{') depth++; - else if (c == ']' || c == '}') depth--; - else if (c == ',' && depth == 0) break; - } - } - - std::string arg_value = args_str.substr(val_start, val_end - val_start); - - if (!quoted) { - if (arg_value == "True") arg_value = "true"; - else if (arg_value == "False") arg_value = "false"; - else if (arg_value == "None") arg_value = "null"; - } - - if (!first_arg) json_call += ","; - json_call += "\"" + arg_name + "\":"; - json_call += quoted ? ("\"" + arg_value + "\"") : arg_value; - first_arg = false; - - arg_pos = args_str.find(',', val_end); - if (arg_pos != std::string::npos) { - arg_pos++; - } else { - break; - } - } - - json_call += "}}"; - function_calls.push_back(json_call); -} - -inline void parse_function_calls_from_response(const std::string& response_text, - std::string& regular_response, - std::vector& function_calls) { - regular_response = response_text; - function_calls.clear(); - - gemma::parse_function_calls(regular_response, function_calls); - - const std::string QWEN_TOOL_START = ""; - const std::string QWEN_TOOL_END = ""; - size_t qwen_start_pos = 0; - - while ((qwen_start_pos = regular_response.find(QWEN_TOOL_START, qwen_start_pos)) != std::string::npos) { - size_t content_start = qwen_start_pos + QWEN_TOOL_START.length(); - size_t qwen_end_pos = regular_response.find(QWEN_TOOL_END, content_start); - - size_t erase_end; - std::string json_content; - - if (qwen_end_pos != std::string::npos) { - json_content = regular_response.substr(content_start, qwen_end_pos - content_start); - erase_end = qwen_end_pos + QWEN_TOOL_END.length(); - } else { - json_content = regular_response.substr(content_start); - erase_end = regular_response.length(); - } - - size_t first = json_content.find_first_not_of(" \t\n\r"); - size_t last = json_content.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) { - json_content = json_content.substr(first, last - first + 1); - } - - if (json_content.size() > 2 && json_content.find("\"name\"") != std::string::npos) { - // Unwrap array wrapper if present: [{"name":...}] -> {"name":...} - if (json_content[0] == '[') { - size_t obj_start = json_content.find('{'); - size_t obj_end = json_content.rfind('}'); - if (obj_start != std::string::npos && obj_end != std::string::npos && obj_end > obj_start) { - json_content = json_content.substr(obj_start, obj_end - obj_start + 1); - } - } - if (json_content[0] == '{') { - size_t depth = 0; - bool in_string = false; - bool escaped = false; - size_t end_pos = 0; - for (size_t c = 0; c < json_content.size(); c++) { - char ch = json_content[c]; - if (escaped) { escaped = false; continue; } - if (ch == '\\' && in_string) { escaped = true; continue; } - if (ch == '"') { in_string = !in_string; continue; } - if (!in_string) { - if (ch == '{') depth++; - else if (ch == '}' && --depth == 0) { end_pos = c + 1; break; } - } - } - if (end_pos > 0) { - function_calls.push_back(json_content.substr(0, end_pos)); - } - } - } - - regular_response.erase(qwen_start_pos, erase_end - qwen_start_pos); - } - - const std::string TOOL_CALL_START = "<|tool_call_start|>"; - const std::string TOOL_CALL_END = "<|tool_call_end|>"; - size_t lfm2_start_pos = 0; - - while ((lfm2_start_pos = regular_response.find(TOOL_CALL_START, lfm2_start_pos)) != std::string::npos) { - size_t content_start = lfm2_start_pos + TOOL_CALL_START.length(); - size_t tool_end_pos = regular_response.find(TOOL_CALL_END, content_start); - - if (tool_end_pos != std::string::npos) { - std::string tool_content = regular_response.substr(content_start, tool_end_pos - content_start); - std::string content = tool_content; - size_t trim_start = 0; - while (trim_start < content.size() && std::isspace(static_cast(content[trim_start]))) { - trim_start++; - } - - if (trim_start < content.size()) { - size_t trim_end = content.size() - 1; - while (trim_end > trim_start && std::isspace(static_cast(content[trim_end]))) { - trim_end--; - } - content = content.substr(trim_start, trim_end - trim_start + 1); - } else { - content.clear(); - } - - if (!content.empty() && content.front() == '[' && content.back() == ']') { - std::string inner = content.substr(1, content.size() - 2); - - size_t inner_first = inner.find_first_not_of(" \t\n\r"); - if (inner_first != std::string::npos && inner[inner_first] == '{') { - size_t pos = inner_first; - while (pos < inner.size()) { - if (inner[pos] == '{') { - int brace_depth = 1; - size_t obj_start = pos; - pos++; - while (pos < inner.size() && brace_depth > 0) { - if (inner[pos] == '{') brace_depth++; - else if (inner[pos] == '}') brace_depth--; - pos++; - } - if (brace_depth == 0) { - std::string json_obj = inner.substr(obj_start, pos - obj_start); - if (json_obj.find("\"name\"") != std::string::npos) { - function_calls.push_back(json_obj); - } - } - } else { - pos++; - } - } - } else { - size_t start = 0; - int paren_depth = 0; - - for (size_t i = 0; i < inner.size(); ++i) { - char c = inner[i]; - if (c == '(') { - paren_depth++; - } else if (c == ')' && paren_depth > 0) { - paren_depth--; - } else if (c == ',' && paren_depth == 0) { - append_lfm2_call(inner.substr(start, i - start), function_calls); - start = i + 1; - } - } - - if (start < inner.size()) { - append_lfm2_call(inner.substr(start), function_calls); - } - } - } else if (!content.empty()) { - append_lfm2_call(content, function_calls); - } - - regular_response.erase(lfm2_start_pos, tool_end_pos + TOOL_CALL_END.length() - lfm2_start_pos); - } else { - break; - } - } - - const char* FUNCTION_CALL_MARKER = "\"function_call\""; - size_t search_pos = 0; - const size_t text_len = regular_response.length(); - - while (search_pos < text_len) { - size_t marker_pos = regular_response.find(FUNCTION_CALL_MARKER, search_pos); - if (marker_pos == std::string::npos) break; - - size_t json_start = regular_response.find('{', marker_pos); - if (json_start == std::string::npos) break; - - int brace_count = 1; - size_t json_end = json_start + 1; - while (json_end < text_len && brace_count > 0) { - char c = regular_response[json_end]; - brace_count += (c == '{') - (c == '}'); - json_end++; - } - - if (brace_count == 0) { - function_calls.push_back(regular_response.substr(json_start, json_end - json_start)); - regular_response = regular_response.substr(0, marker_pos); - size_t last_bracket = regular_response.rfind('{'); - if(last_bracket != std::string::npos) { - regular_response = regular_response.substr(0, last_bracket); - } - } - search_pos = json_end; - } -} - -inline std::vector> find_channel_token_ranges( - const std::vector& tokens, size_t offset, - uint32_t channel_open_id, uint32_t channel_close_id) { - std::vector> ranges; - size_t pos = 0; - while (pos < tokens.size()) { - if (tokens[pos] != channel_open_id) { - pos++; - continue; - } - - size_t block_start = pos; - pos++; - while (pos < tokens.size() && tokens[pos] != channel_close_id) { - pos++; - } - if (pos < tokens.size()) { - pos++; - } - ranges.push_back({offset + block_start, pos - block_start}); - } - return ranges; -} - -inline void strip_tag_blocks(std::string& text, std::string& extracted, - const std::string& open_tag, const std::string& close_tag) { - std::string result; - size_t pos = 0; - - size_t first_close = text.find(close_tag); - size_t first_open = text.find(open_tag); - if (first_close != std::string::npos && - (first_open == std::string::npos || first_close < first_open)) { - extracted += text.substr(0, first_close); - pos = first_close + close_tag.size(); - } - - while (pos < text.size()) { - size_t open_pos = text.find(open_tag, pos); - if (open_pos == std::string::npos) { - result += text.substr(pos); - break; - } - result += text.substr(pos, open_pos - pos); - size_t content_start = open_pos + open_tag.size(); - size_t close_pos = text.find(close_tag, content_start); - if (close_pos == std::string::npos) { - if (!extracted.empty()) extracted += "\n"; - extracted += text.substr(content_start); - break; - } - if (!extracted.empty()) extracted += "\n"; - extracted += text.substr(content_start, close_pos - content_start); - pos = close_pos + close_tag.size(); - } - text = result; -} - -inline void strip_thinking_block(const std::string& input, std::string& thinking, std::string& content) { - thinking.clear(); - content = input; - - auto trim = [](std::string& s) { - size_t first = s.find_first_not_of(" \t\n\r"); - size_t last = s.find_last_not_of(" \t\n\r"); - if (first != std::string::npos && last != std::string::npos) - s = s.substr(first, last - first + 1); - else - s.clear(); - }; - - if (content.find("<|channel>") != std::string::npos || content.find("") != std::string::npos) { - strip_tag_blocks(content, thinking, "<|channel>", ""); - } else if (content.find("") != std::string::npos || content.find("") != std::string::npos) { - strip_tag_blocks(content, thinking, "", ""); - } else { - return; - } - - trim(thinking); - trim(content); -} - -struct TranscriptSegment { - float start; - float end; - std::string text; -}; - -inline std::string construct_response_json(const std::string& regular_response, - const std::vector& function_calls, - double time_to_first_token, - double total_time_ms, - double prefill_tps, - double decode_tps, - size_t prompt_tokens, - size_t completion_tokens, - float confidence = 0.0f, - bool cloud_handoff = false, - const std::string& thinking = "", - const std::vector& segments = {}) { - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"cloud_handoff\":" << (cloud_handoff ? "true" : "false") << ","; - json << "\"response\":\"" << escape_json_string(regular_response) << "\","; - if (!thinking.empty()) { - json << "\"thinking\":\"" << escape_json_string(thinking) << "\","; - } - json << "\"function_calls\":["; - for (size_t i = 0; i < function_calls.size(); ++i) { - if (i > 0) json << ","; - json << function_calls[i]; - } - json << "],"; - json << "\"segments\":["; - for (size_t i = 0; i < segments.size(); ++i) { - if (i > 0) json << ","; - json << "{\"start\":" << std::fixed << std::setprecision(3) << segments[i].start - << ",\"end\":" << std::fixed << std::setprecision(3) << segments[i].end - << ",\"text\":\"" << escape_json_string(segments[i].text) << "\"}"; - } - json << "],"; - json << "\"confidence\":" << std::fixed << std::setprecision(4) << confidence << ","; - json << "\"time_to_first_token_ms\":" << std::fixed << std::setprecision(2) << time_to_first_token << ","; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"prefill_tps\":" << std::fixed << std::setprecision(2) << prefill_tps << ","; - json << "\"decode_tps\":" << std::fixed << std::setprecision(2) << decode_tps << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb() << ","; - json << "\"prefill_tokens\":" << prompt_tokens << ","; - json << "\"decode_tokens\":" << completion_tokens << ","; - json << "\"total_tokens\":" << (prompt_tokens + completion_tokens); - json << "}"; - return json.str(); -} - -inline std::string serialize_function_calls(const std::vector& calls) { - if (calls.empty()) return "[]"; - std::ostringstream oss; - oss << "["; - for (size_t i = 0; i < calls.size(); ++i) { - if (i > 0) oss << ","; - oss << calls[i]; - } - oss << "]"; - return oss.str(); -} - -inline int validate_audio_params( - const char* component, - void* model, - char* response_buffer, size_t buffer_size, - const char* audio_file_path, - const uint8_t* pcm_buffer, size_t pcm_buffer_size) { - if (!model) { - std::string err = last_error_message.empty() ? "Model not initialized." : last_error_message; - CACTUS_LOG_ERROR(component, err); - handle_error_response(err, response_buffer, buffer_size); - return -1; - } - if (!response_buffer || buffer_size == 0) { - CACTUS_LOG_ERROR(component, "Invalid parameters: response_buffer or buffer_size"); - handle_error_response("Invalid parameters", response_buffer, buffer_size); - return -1; - } - if (!audio_file_path && (!pcm_buffer || pcm_buffer_size == 0)) { - CACTUS_LOG_ERROR(component, "No audio input provided"); - handle_error_response("Either audio_file_path or pcm_buffer must be provided", response_buffer, buffer_size); - return -1; - } - if (audio_file_path && pcm_buffer && pcm_buffer_size > 0) { - CACTUS_LOG_ERROR(component, "Both audio_file_path and pcm_buffer provided"); - handle_error_response("Cannot provide both audio_file_path and pcm_buffer", response_buffer, buffer_size); - return -1; - } - if (pcm_buffer && pcm_buffer_size > 0 && (pcm_buffer_size < 2 || pcm_buffer_size % 2 != 0)) { - CACTUS_LOG_ERROR(component, "Invalid pcm_buffer_size"); - handle_error_response("pcm_buffer_size must be even and at least 2 bytes", response_buffer, buffer_size); - return -1; - } - return 0; -} - -inline std::vector pcm_to_float(const uint8_t* pcm_buffer, size_t pcm_buffer_size) { - const int16_t* samples = reinterpret_cast(pcm_buffer); - size_t n = pcm_buffer_size / 2; - std::vector out(n); - for (size_t i = 0; i < n; ++i) - out[i] = static_cast(samples[i]) / 32768.0f; - return out; -} - -} // namespace ffi -} // namespace cactus - -#ifdef __cplusplus -extern "C" { -#endif - -const char* cactus_get_last_error(); - -#ifdef __cplusplus -} -#endif - -#endif // CACTUS_UTILS_H diff --git a/cactus/ffi/cactus_vad.cpp b/cactus/ffi/cactus_vad.cpp deleted file mode 100644 index 10b2e99c3..000000000 --- a/cactus/ffi/cactus_vad.cpp +++ /dev/null @@ -1,176 +0,0 @@ -#include "cactus_ffi.h" -#include "cactus_utils.h" -#include "../models/model.h" -#include "../../libs/audio/wav.h" -#include -#include - -using namespace cactus::engine; -using namespace cactus::ffi; - - -static SileroVADModel::SpeechTimestampsOptions parse_speech_timestamps_options(const std::string& json) { - SileroVADModel::SpeechTimestampsOptions options; - - if (json.empty()) return options; - - size_t pos = json.find("\"threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"neg_threshold\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.neg_threshold = std::stof(json.substr(pos)); - } - - pos = json.find("\"min_speech_duration_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_speech_duration_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"max_speech_duration_s\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.max_speech_duration_s = std::stof(json.substr(pos)); - } - - pos = json.find("\"min_silence_duration_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_silence_duration_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"speech_pad_ms\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.speech_pad_ms = std::stoi(json.substr(pos)); - } - - pos = json.find("\"window_size_samples\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.window_size_samples = std::stoi(json.substr(pos)); - } - - pos = json.find("\"min_silence_at_max_speech\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.min_silence_at_max_speech = std::stoi(json.substr(pos)); - } - - pos = json.find("\"use_max_poss_sil_at_max_speech\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - std::string value = json.substr(pos, 5); - options.use_max_poss_sil_at_max_speech = (value.find("true") != std::string::npos); - } - - pos = json.find("\"sampling_rate\""); - if (pos != std::string::npos) { - pos = json.find(':', pos) + 1; - while (pos < json.length() && std::isspace(json[pos])) pos++; - options.sampling_rate = std::stoi(json.substr(pos)); - } - - return options; -} - - -extern "C" { - -int cactus_vad( - cactus_model_t model, - const char* audio_file_path, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size -) { - if (validate_audio_params("vad", model, response_buffer, buffer_size, audio_file_path, pcm_buffer, pcm_buffer_size) != 0) - return -1; - - try { - auto start_time = std::chrono::high_resolution_clock::now(); - auto* handle = static_cast(model); - auto* vad_model = static_cast(handle->model.get()); - - SileroVADModel::SpeechTimestampsOptions options = parse_speech_timestamps_options(options_json ? options_json : ""); - - std::vector audio; - if (audio_file_path == nullptr) { - audio = pcm_to_float(pcm_buffer, pcm_buffer_size); - } else { - AudioFP32 wav_audio = load_wav(audio_file_path); - audio = resample_to_16k_fp32(wav_audio.samples, wav_audio.sample_rate); - } - - if (audio.empty()) { - last_error_message = "Failed to load audio or audio is empty"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - auto segments = vad_model->get_speech_timestamps(audio, options); - - auto end_time = std::chrono::high_resolution_clock::now(); - double total_time_ms = std::chrono::duration_cast(end_time - start_time).count() / 1000.0; - - std::ostringstream json; - json << "{"; - json << "\"success\":true,"; - json << "\"error\":null,"; - json << "\"segments\":["; - - for (size_t i = 0; i < segments.size(); ++i) { - if (i > 0) json << ","; - json << "{"; - json << "\"start\":" << segments[i].start << ","; - json << "\"end\":" << segments[i].end; - json << "}"; - } - - json << "],"; - json << "\"total_time_ms\":" << std::fixed << std::setprecision(2) << total_time_ms << ","; - json << "\"ram_usage_mb\":" << std::fixed << std::setprecision(2) << get_ram_usage_mb(); - json << "}"; - - std::string response = json.str(); - if (response.length() >= buffer_size) { - last_error_message = "Response buffer too small"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } - - std::strcpy(response_buffer, response.c_str()); - return static_cast(response.length()); - - } catch (const std::exception& e) { - last_error_message = "Exception during VAD processing: " + std::string(e.what()); - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } catch (...) { - last_error_message = "Unknown exception during VAD processing"; - CACTUS_LOG_ERROR("vad", last_error_message); - handle_error_response(last_error_message, response_buffer, buffer_size); - return -1; - } -} - -} diff --git a/cactus/ffi/gemma_tools.h b/cactus/ffi/gemma_tools.h deleted file mode 100644 index f0f9fe26d..000000000 --- a/cactus/ffi/gemma_tools.h +++ /dev/null @@ -1,576 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace gemma { - -inline std::string to_upper(const std::string& s) { - std::string result = s; - for (auto& c : result) c = std::toupper(c); - return result; -} - -inline std::string escape(const std::string& s) { - return "" + s + ""; -} - -inline void skip_whitespace(const std::string& json, size_t& pos) { - while (pos < json.length() && std::isspace(json[pos])) pos++; -} - -inline std::string extract_json_string(const std::string& json, size_t& pos) { - std::string value; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\' && pos + 1 < json.length()) { - pos++; - if (json[pos] == 'n') value += '\n'; - else if (json[pos] == 't') value += '\t'; - else if (json[pos] == 'r') value += '\r'; - else if (json[pos] == '"') value += '"'; - else if (json[pos] == '\\') value += '\\'; - else value += json[pos]; - } else { - value += json[pos]; - } - pos++; - } - if (pos < json.length()) pos++; - return value; -} - -std::string format_argument(const std::string& json, size_t& pos, bool escape_keys); -std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/); - -inline std::string format_argument(const std::string& json, size_t& pos, bool escape_keys = true) { - skip_whitespace(json, pos); - if (pos >= json.length()) return ""; - - char c = json[pos]; - - if (c == '"') { - pos++; - std::string value = extract_json_string(json, pos); - return escape(value); - } else if (c == '{') { - std::string result = "{"; - pos++; - bool first = true; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == '}') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - if (json[pos] != '"') break; - pos++; - std::string key = extract_json_string(json, pos); - - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == ':') pos++; - - std::string value = format_argument(json, pos, escape_keys); - - if (!first) result += ","; - first = false; - if (escape_keys) { - result += escape(key) + ":" + value; - } else { - result += key + ":" + value; - } - } - result += "}"; - return result; - } else if (c == '[') { - std::string result = "["; - pos++; - bool first = true; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == ']') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - std::string value = format_argument(json, pos, escape_keys); - - if (!first) result += ","; - first = false; - result += value; - } - result += "]"; - return result; - } else if (json.compare(pos, 4, "true") == 0) { - pos += 4; - return "true"; - } else if (json.compare(pos, 5, "false") == 0) { - pos += 5; - return "false"; - } else if (json.compare(pos, 4, "null") == 0) { - pos += 4; - return "null"; - } else { - size_t start = pos; - while (pos < json.length() && (std::isdigit(json[pos]) || json[pos] == '.' || - json[pos] == '-' || json[pos] == '+' || json[pos] == 'e' || json[pos] == 'E')) { - pos++; - } - return json.substr(start, pos - start); - } -} - -inline std::map parse_json_object_raw(const std::string& json, size_t& pos) { - std::map result; - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] != '{') return result; - pos++; - - while (pos < json.length()) { - skip_whitespace(json, pos); - if (pos >= json.length() || json[pos] == '}') { pos++; break; } - if (json[pos] == ',') { pos++; continue; } - - if (json[pos] != '"') break; - pos++; - std::string key = extract_json_string(json, pos); - - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == ':') pos++; - skip_whitespace(json, pos); - - size_t value_start = pos; - if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - pos++; - } else if (json[pos] == '{') { - int depth = 1; - pos++; - while (pos < json.length() && depth > 0) { - if (json[pos] == '{') depth++; - else if (json[pos] == '}') depth--; - else if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - } else if (json[pos] == '[') { - int depth = 1; - pos++; - while (pos < json.length() && depth > 0) { - if (json[pos] == '[') depth++; - else if (json[pos] == ']') depth--; - else if (json[pos] == '"') { - pos++; - while (pos < json.length() && json[pos] != '"') { - if (json[pos] == '\\') pos++; - pos++; - } - } - pos++; - } - } else { - while (pos < json.length() && json[pos] != ',' && json[pos] != '}') pos++; - } - result[key] = json.substr(value_start, pos - value_start); - } - return result; -} - -inline std::string get_json_string_value(const std::string& json, size_t pos) { - skip_whitespace(json, pos); - if (pos < json.length() && json[pos] == '"') { - pos++; - return extract_json_string(json, pos); - } - return ""; -} - -inline std::string format_parameters(const std::string& properties_json, const std::string& /*required_json*/) { - static const std::set standard_keys = {"description", "type", "properties", "required", "nullable"}; - - size_t pos = 0; - auto properties = parse_json_object_raw(properties_json, pos); - - std::string result; - bool first = true; - - for (const auto& [key, value_json] : properties) { - if (standard_keys.count(key)) continue; - - if (!first) result += ","; - first = false; - - size_t prop_pos = 0; - auto prop_obj = parse_json_object_raw(value_json, prop_pos); - - result += key + ":{"; - - if (prop_obj.count("description")) { - std::string desc = get_json_string_value(prop_obj["description"], 0); - result += "description:" + escape(desc); - } - - std::string type_val; - if (prop_obj.count("type")) { - type_val = get_json_string_value(prop_obj["type"], 0); - } - - if (to_upper(type_val) == "STRING") { - if (prop_obj.count("enum")) { - size_t enum_pos = 0; - std::string enum_formatted = format_argument(prop_obj["enum"], enum_pos, true); - result += ",enum:" + enum_formatted; - } - } else if (to_upper(type_val) == "OBJECT") { - if (prop_obj.count("properties")) { - std::string nested_required; - if (prop_obj.count("required")) { - nested_required = prop_obj["required"]; - } - result += ",properties:{" + format_parameters(prop_obj["properties"], nested_required) + "}"; - } - if (prop_obj.count("required")) { - std::string req_items; - size_t req_pos = 0; - skip_whitespace(prop_obj["required"], req_pos); - if (req_pos < prop_obj["required"].length() && prop_obj["required"][req_pos] == '[') { - req_pos++; - bool req_first = true; - while (req_pos < prop_obj["required"].length()) { - skip_whitespace(prop_obj["required"], req_pos); - if (prop_obj["required"][req_pos] == ']') break; - if (prop_obj["required"][req_pos] == ',') { req_pos++; continue; } - if (prop_obj["required"][req_pos] == '"') { - req_pos++; - std::string req_item = extract_json_string(prop_obj["required"], req_pos); - if (!req_first) req_items += ","; - req_first = false; - req_items += escape(req_item); - } - } - } - if (!req_items.empty()) { - result += ",required:[" + req_items + "]"; - } - } - } else if (to_upper(type_val) == "ARRAY") { - if (prop_obj.count("items")) { - result += ",items:{"; - size_t items_pos = 0; - auto items_obj = parse_json_object_raw(prop_obj["items"], items_pos); - bool items_first = true; - - for (const auto& [item_key, item_value] : items_obj) { - if (!items_first) result += ","; - items_first = false; - - if (item_key == "properties") { - std::string items_required; - if (items_obj.count("required")) { - items_required = items_obj["required"]; - } - result += "properties:{" + format_parameters(item_value, items_required) + "}"; - } else if (item_key == "required") { - result += "required:["; - size_t req_pos = 0; - skip_whitespace(item_value, req_pos); - if (req_pos < item_value.length() && item_value[req_pos] == '[') { - req_pos++; - bool req_first = true; - while (req_pos < item_value.length()) { - skip_whitespace(item_value, req_pos); - if (item_value[req_pos] == ']') break; - if (item_value[req_pos] == ',') { req_pos++; continue; } - if (item_value[req_pos] == '"') { - req_pos++; - std::string req_item = extract_json_string(item_value, req_pos); - if (!req_first) result += ","; - req_first = false; - result += escape(req_item); - } - } - } - result += "]"; - } else if (item_key == "type") { - std::string item_type = get_json_string_value(item_value, 0); - result += "type:" + escape(to_upper(item_type)); - } else { - size_t val_pos = 0; - result += item_key + ":" + format_argument(item_value, val_pos, true); - } - } - result += "}"; - } - } - - if (!type_val.empty()) { - result += ",type:" + escape(to_upper(type_val)); - } - - result += "}"; - } - - return result; -} - -inline std::string format_function_declaration(const std::string& name, - const std::string& description, - const std::string& params_json) { - std::string result = "declaration:" + name + "{"; - result += "description:" + escape(description); - - if (!params_json.empty()) { - result += ",parameters:{"; - - size_t pos = 0; - auto params = parse_json_object_raw(params_json, pos); - - if (params.count("properties")) { - std::string required_json; - if (params.count("required")) { - required_json = params["required"]; - } - result += "properties:{" + format_parameters(params["properties"], required_json) + "}"; - } - - if (params.count("required")) { - std::string req_items; - size_t req_pos = 0; - skip_whitespace(params["required"], req_pos); - if (req_pos < params["required"].length() && params["required"][req_pos] == '[') { - req_pos++; - bool first = true; - while (req_pos < params["required"].length()) { - skip_whitespace(params["required"], req_pos); - if (params["required"][req_pos] == ']') break; - if (params["required"][req_pos] == ',') { req_pos++; continue; } - if (params["required"][req_pos] == '"') { - req_pos++; - std::string item = extract_json_string(params["required"], req_pos); - if (!first) req_items += ","; - first = false; - req_items += escape(item); - } - } - } - if (!req_items.empty()) { - result += ",required:[" + req_items + "]"; - } - } - - if (params.count("type")) { - std::string type_val = get_json_string_value(params["type"], 0); - result += ",type:" + escape(to_upper(type_val)); - } - - result += "}"; - } - - result += "}"; - return result; -} - -template -inline std::string format_tools(const std::vector& tools, bool use_pipe_tags = false) { - if (tools.empty()) return ""; - - const char* decl_start = use_pipe_tags ? "<|tool>" : ""; - const char* decl_end = use_pipe_tags ? "" : ""; - - std::string result; - for (const auto& tool : tools) { - result += decl_start; - std::string params_json; - auto it = tool.parameters.find("schema"); - if (it != tool.parameters.end()) { - params_json = it->second; - } - - result += format_function_declaration(tool.name, tool.description, params_json); - result += decl_end; - } - return result; -} - - -inline size_t match_quote_tag(const std::string& s, size_t pos) { - if (s.compare(pos, 8, "") == 0) return 8; - if (s.compare(pos, 5, "<|\"|>") == 0) return 5; - return 0; -} - -inline size_t find_quote_tag(const std::string& s, size_t pos) { - size_t e = s.find("", pos); - size_t t = s.find("<|\"|>", pos); - if (e == std::string::npos) return t; - if (t == std::string::npos) return e; - return std::min(e, t); -} - -inline std::string unescape(const std::string& s) { - const std::string ESCAPE_TAG = ""; - std::string result = s; - size_t pos = 0; - while ((pos = result.find(ESCAPE_TAG, pos)) != std::string::npos) { - result.erase(pos, ESCAPE_TAG.length()); - } - return result; -} - -inline std::string args_to_json(const std::string& args_content) { - std::string result = "{"; - size_t pos = 0; - bool first = true; - - if (!args_content.empty() && args_content[0] == '{') pos = 1; - - while (pos < args_content.length()) { - while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; - if (pos >= args_content.length() || args_content[pos] == '}') break; - if (args_content[pos] == ',') { pos++; continue; } - - size_t key_start = pos; - while (pos < args_content.length() && args_content[pos] != ':') pos++; - std::string key = args_content.substr(key_start, pos - key_start); - if (pos < args_content.length()) pos++; - - std::string value; - while (pos < args_content.length() && std::isspace(args_content[pos])) pos++; - - if (pos < args_content.length()) { - size_t qtag_len = match_quote_tag(args_content, pos); - if (qtag_len > 0) { - pos += qtag_len; - size_t val_end = find_quote_tag(args_content, pos); - if (val_end != std::string::npos) { - value = "\"" + args_content.substr(pos, val_end - pos) + "\""; - pos = val_end + match_quote_tag(args_content, val_end); - } - } else if (args_content[pos] == '{') { - int depth = 1; - size_t start = pos; - pos++; - while (pos < args_content.length() && depth > 0) { - if (args_content[pos] == '{') depth++; - else if (args_content[pos] == '}') depth--; - pos++; - } - value = args_to_json(args_content.substr(start, pos - start)); - } else if (args_content[pos] == '[') { - int depth = 1; - size_t start = pos; - pos++; - while (pos < args_content.length() && depth > 0) { - if (args_content[pos] == '[') depth++; - else if (args_content[pos] == ']') depth--; - pos++; - } - std::string arr_content = args_content.substr(start + 1, pos - start - 2); - value = "["; - size_t arr_pos = 0; - bool first_item = true; - while (arr_pos < arr_content.length()) { - while (arr_pos < arr_content.length() && (std::isspace(arr_content[arr_pos]) || arr_content[arr_pos] == ',')) arr_pos++; - if (arr_pos >= arr_content.length()) break; - - if (!first_item) value += ","; - first_item = false; - - size_t aq_len = match_quote_tag(arr_content, arr_pos); - if (aq_len > 0) { - arr_pos += aq_len; - size_t end = find_quote_tag(arr_content, arr_pos); - if (end != std::string::npos) { - value += "\"" + arr_content.substr(arr_pos, end - arr_pos) + "\""; - arr_pos = end + match_quote_tag(arr_content, end); - } - } else { - size_t end = arr_content.find_first_of(",]", arr_pos); - if (end == std::string::npos) end = arr_content.length(); - value += arr_content.substr(arr_pos, end - arr_pos); - arr_pos = end; - } - } - value += "]"; - } else { - size_t val_start = pos; - while (pos < args_content.length() && args_content[pos] != ',' && args_content[pos] != '}') { - pos++; - } - value = args_content.substr(val_start, pos - val_start); - while (!value.empty() && std::isspace(value.back())) value.pop_back(); - } - } - - if (!first) result += ","; - first = false; - result += "\"" + key + "\":" + value; - } - - result += "}"; - return result; -} - -inline void parse_function_calls(std::string& response, std::vector& function_calls) { - - const std::string CALL_START = (response.find("<|tool_call>") != std::string::npos) - ? "<|tool_call>" : ""; - const std::string CALL_END = (CALL_START == "<|tool_call>") - ? "" : ""; - size_t pos = 0; - - while ((pos = response.find(CALL_START, pos)) != std::string::npos) { - size_t content_start = pos + CALL_START.length(); - size_t call_end_pos = response.find(CALL_END, content_start); - - size_t content_end = (call_end_pos != std::string::npos) ? call_end_pos : response.length(); - std::string call_content = response.substr(content_start, content_end - content_start); - - if (call_content.compare(0, 5, "call:") == 0) { - size_t brace_pos = call_content.find('{'); - - if (brace_pos == std::string::npos) { - size_t sep_pos = call_content.find_first_of(", ", 5); - if (sep_pos != std::string::npos) { - std::string func_name = call_content.substr(5, sep_pos - 5); - size_t args_start = sep_pos + 1; - while (args_start < call_content.length() && - (call_content[args_start] == ' ' || call_content[args_start] == ',')) { - args_start++; - } - std::string args_content = "{" + call_content.substr(args_start); - if (args_content.back() != '}') args_content += "}"; - - std::string args_json = args_to_json(args_content); - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; - function_calls.push_back(json_call); - } - } else { - std::string func_name = call_content.substr(5, brace_pos - 5); - std::string args_content = call_content.substr(brace_pos); - if (args_content.back() != '}') args_content += "}"; - - std::string args_json = args_to_json(args_content); - std::string json_call = "{\"name\":\"" + func_name + "\",\"arguments\":" + args_json + "}"; - function_calls.push_back(json_call); - } - } - - size_t erase_end = (call_end_pos != std::string::npos) ? - call_end_pos + CALL_END.length() : response.length(); - response.erase(pos, erase_end - pos); - } -} - -} // namespace gemma \ No newline at end of file diff --git a/cactus/ffi/needle_tools.h b/cactus/ffi/needle_tools.h deleted file mode 100644 index 4330cff51..000000000 --- a/cactus/ffi/needle_tools.h +++ /dev/null @@ -1,87 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace needle { - -inline std::string to_snake_case(const std::string& name) { - std::string s; - s.reserve(name.size() * 2); - for (char c : name) { - if (std::isalnum(static_cast(c)) || c == '_') { - s += c; - } else { - if (s.empty() || s.back() != '_') s += '_'; - } - } - - std::string s2; - s2.reserve(s.size() * 2); - for (size_t i = 0; i < s.size(); i++) { - char c = s[i]; - if (i > 0 && std::isupper(static_cast(c))) { - char prev = s[i - 1]; - if (std::islower(static_cast(prev)) || std::isdigit(static_cast(prev))) { - s2 += '_'; - } - } - s2 += c; - } - - std::string s3; - s3.reserve(s2.size() * 2); - for (size_t i = 0; i < s2.size(); i++) { - s3 += s2[i]; - if (i + 1 < s2.size() && - std::isupper(static_cast(s2[i])) && - std::isupper(static_cast(s2[i + 1]))) { - if (i + 2 < s2.size() && std::islower(static_cast(s2[i + 2]))) { - s3 += '_'; - } - } - } - - std::string result; - result.reserve(s3.size()); - bool prev_underscore = false; - for (char c : s3) { - if (c == '_') { - if (!prev_underscore) result += '_'; - prev_underscore = true; - } else { - result += static_cast(std::tolower(static_cast(c))); - prev_underscore = false; - } - } - - size_t start = result.find_first_not_of('_'); - if (start == std::string::npos) return result; - size_t end = result.find_last_not_of('_'); - return result.substr(start, end - start + 1); -} - -inline void restore_tool_names(std::vector& function_calls, - const std::unordered_map& name_map) { - if (name_map.empty()) return; - for (auto& call : function_calls) { - for (const auto& [snake, orig] : name_map) { - std::string from = "\"name\":\"" + snake + "\""; - size_t pos = call.find(from); - if (pos == std::string::npos) { - from = "\"name\": \"" + snake + "\""; - pos = call.find(from); - } - if (pos != std::string::npos) { - std::string to = from.substr(0, from.size() - snake.size() - 1) + orig + "\""; - call.replace(pos, from.size(), to); - break; - } - } - } -} - -} // namespace needle diff --git a/cactus/graph/graph.h b/cactus/graph/graph.h deleted file mode 100644 index b43617758..000000000 --- a/cactus/graph/graph.h +++ /dev/null @@ -1,779 +0,0 @@ -#ifndef GRAPH_H -#define GRAPH_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { - -enum class LogLevel { - DEBUG = 0, - INFO = 1, - WARN = 2, - ERROR = 3, - NONE = 4 -}; - -class Logger { -public: - static Logger& instance() { - static Logger logger; - return logger; - } - - void set_level(LogLevel level) { min_level_ = level; } - LogLevel get_level() const { return min_level_; } - - void set_callback(std::function cb) { - std::lock_guard lock(mutex_); - callback_ = cb; - } - - void log(LogLevel level, const std::string& component, const std::string& message) { - if (level < min_level_) return; - - std::lock_guard lock(mutex_); - - if (callback_) { - callback_(level, component, message); - } else { - std::cerr << "[" << level_string(level) << "] [" << component << "] " << message << std::endl; - } - - if (level == LogLevel::ERROR) { - last_error_ = "[" + component + "] " + message; - } - } - - const std::string& last_error() const { return last_error_; } - void clear_error() { last_error_.clear(); } - -private: - Logger() : min_level_(LogLevel::WARN) {} - - static const char* level_string(LogLevel level) { - switch (level) { - case LogLevel::DEBUG: return "DEBUG"; - case LogLevel::INFO: return "INFO"; - case LogLevel::WARN: return "WARN"; - case LogLevel::ERROR: return "ERROR"; - default: return "?"; - } - } - - LogLevel min_level_; - std::mutex mutex_; - std::string last_error_; - std::function callback_; -}; - -} // namespace cactus - -#define CACTUS_LOG(level, component, msg) \ - do { \ - if (static_cast(level) >= static_cast(cactus::Logger::instance().get_level())) { \ - std::ostringstream _cactus_log_ss; \ - _cactus_log_ss << msg; \ - cactus::Logger::instance().log(level, component, _cactus_log_ss.str()); \ - } \ - } while(0) - -#define CACTUS_LOG_DEBUG(component, msg) CACTUS_LOG(cactus::LogLevel::DEBUG, component, msg) -#define CACTUS_LOG_INFO(component, msg) CACTUS_LOG(cactus::LogLevel::INFO, component, msg) -#define CACTUS_LOG_WARN(component, msg) CACTUS_LOG(cactus::LogLevel::WARN, component, msg) -#define CACTUS_LOG_ERROR(component, msg) CACTUS_LOG(cactus::LogLevel::ERROR, component, msg) - -namespace GraphFile { - class MappedFile; - struct SerializedGraph; -} - -enum class Precision { - INT8, - FP16, - FP32, - INT4 -}; - -enum class ComputeBackend { - CPU, - NPU -}; - -enum class Activation { - SILU, - GELU, - GELU_ERF, - RELU, - SIGMOID, - TANH -}; - -enum class OpType { - INPUT, PRECISION_CAST, - ADD, ADD_CLIPPED, SUBTRACT, MULTIPLY, DIVIDE, - ABS, POW, FLATTEN, VIEW, - MATMUL, TRANSPOSE, RESHAPE, SLICE, GATHER, EMBEDDING, - BILINEAR_INTERPOLATION, - SUM, MEAN, VARIANCE, MIN, MAX, - RMS_NORM, ROPE, ROPE_GPTJ, SOFTMAX, ATTENTION, ATTENTION_INT8_HYBRID, REL_POS_BIAS, CONV1D_CAUSAL, CONV1D_K3, CONV1D_K7S3, CONV1D, CONV1D_SAME_DEPTHWISE_K9, CONV1D_POINTWISE, CONV2D_K3S2P1, CONV2D_DEPTHWISE_K3S2P1, CONV2D_POINTWISE_1X1, GLU, BATCHNORM, - SCALAR_ADD, SCALAR_SUBTRACT, SCALAR_MULTIPLY, SCALAR_DIVIDE, SCALAR_EXP, SCALAR_SQRT, SCALAR_COS, SCALAR_SIN, SCALAR_LOG, - RELU, SILU, GELU, GELU_ERF, SIGMOID, TANH, - SAMPLE, CONCAT, CAT, - SCATTER_TOPK, - TOPK, LAYERNORM, GROUPNORM, - MOE_LAYER, - INDEX, - PERSISTENT, - QUANTIZE_ACTIVATIONS, - LSTM_CELL, - GATED_DELTANET_DECODE, - GATED_DELTANET_PREFILL, - STFT, - ALTUP_PREDICT, - ALTUP_CORRECT, - GAUSSIAN_TOPK, - MAXPOOL1D, - BILSTM_SEQUENCE, - LEAKY_RELU, - CONV2D_K3S1P1, - STATS_POOL, - WEIGHTED_STATS_POOL, - DENSE_MLP_INT4_FUSED -}; - -struct PrecisionTraits { - static constexpr size_t size_of(Precision prec) { - switch (prec) { - case Precision::INT8: return 1; - case Precision::FP16: return 2; - case Precision::FP32: return 4; - case Precision::INT4: return 1; - } - return 1; - } - - static constexpr size_t packed_size_of(Precision prec, size_t count) { - switch (prec) { - case Precision::INT4: return (count + 1) / 2; - default: return count * size_of(prec); - } - } - - static size_t byte_offset_of(Precision prec, size_t element_offset) { - switch (prec) { - case Precision::INT4: - assert(element_offset % 32 == 0 && "INT4 byte offset must be group-aligned (multiple of 32)"); - return element_offset / 2; - default: return element_offset * size_of(prec); - } - } - - static constexpr bool is_integer(Precision prec) { - switch (prec) { - case Precision::INT8: return true; - case Precision::INT4: return true; - case Precision::FP16: return false; - case Precision::FP32: return false; - } - return true; - } - - static constexpr bool is_floating_point(Precision prec) { - switch (prec) { - case Precision::INT8: return false; - case Precision::INT4: return false; - case Precision::FP16: return true; - case Precision::FP32: return true; - } - return false; - } -}; - -namespace Quantization { - void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); - void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); - void fp16_to_fp32(const __fp16* src, float* dst, size_t count); - void fp32_to_fp16(const float* src, __fp16* dst, size_t count); - void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); - void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); -} - -struct TensorConfig { - Precision default_precision = Precision::INT8; - Precision compute_precision = Precision::INT8; - Precision output_precision = Precision::INT8; - bool auto_mixed_precision = false; - - static TensorConfig& global(); -}; - -struct BroadcastInfo { - std::vector output_shape; - bool needs_broadcasting; - - static BroadcastInfo compute(const std::vector& lhs, const std::vector& rhs); -}; - -class BufferPool; - -struct BufferDesc { - std::vector shape; - size_t total_size; - size_t byte_size; - std::unique_ptr data; - void* external_data; - char* pooled_data; - Precision precision; - - size_t group_size = 0; - size_t num_groups = 0; - void* scales_data = nullptr; - std::unique_ptr owned_scales; - - bool is_interleaved = false; - size_t original_N = 0; - - void* activation_scales_data = nullptr; - std::unique_ptr owned_activation_scales; - size_t num_rows_for_activation_scales = 0; - - BufferDesc(); - BufferDesc(const std::vector& s, Precision prec = Precision::INT8); - ~BufferDesc(); - - BufferDesc(BufferDesc&& other) noexcept; - BufferDesc& operator=(BufferDesc&& other) noexcept; - - BufferDesc(const BufferDesc&) = delete; - BufferDesc& operator=(const BufferDesc&) = delete; - - void* get_data(); - const void* get_data() const; - - template - T* data_as() { return static_cast(get_data()); } - - template - const T* data_as() const { return static_cast(get_data()); } - - const __fp16* scales_as_fp16() const { - return reinterpret_cast(scales_data); - } - - bool is_grouped_int8() const { - return precision == Precision::INT8 && group_size > 0; - } - - bool is_grouped_int4() const { - return precision == Precision::INT4 && group_size > 0; - } - - void set_grouped_scales(size_t gs, size_t ng, void* scales_ptr) { - group_size = gs; - num_groups = ng; - scales_data = scales_ptr; - } - - void set_interleaved(bool interleaved, size_t orig_n) { - is_interleaved = interleaved; - original_N = orig_n; - } - - bool has_activation_scales() const { - return activation_scales_data != nullptr && num_rows_for_activation_scales > 0; - } - const float* activation_scales_as_float() const { - return reinterpret_cast(activation_scales_data); - } - float* activation_scales_as_float() { - return reinterpret_cast(activation_scales_data); - } - void allocate_activation_scales(size_t num_rows) { - num_rows_for_activation_scales = num_rows; - owned_activation_scales = std::make_unique(num_rows * sizeof(float)); - activation_scales_data = owned_activation_scales.get(); - } - void set_activation_scales(void* scales_ptr, size_t num_rows) { - activation_scales_data = scales_ptr; - num_rows_for_activation_scales = num_rows; - } - - void allocate(); - void allocate_from_pool(BufferPool& pool); - void release_to_pool(BufferPool& pool); - void set_external(void* ptr); -}; - -struct OpParams { - float scalar = 0.0f; - float scale = 1.0f; - float theta = 10000.0f; - float epsilon = 1e-6f; - int axis = -1; - bool pretransposed_rhs = false; - size_t position_offset = 0; - size_t slice_start = 0; - size_t slice_length = 0; - size_t window_size = 0; - bool is_causal = true; - bool attention_mask_is_additive = false; - float logit_cap = 0.0f; - std::vector new_shape; - std::vector permutation; - Precision output_precision = Precision::INT8; - BroadcastInfo broadcast_info; - ComputeBackend backend = ComputeBackend::CPU; - - size_t dilation = 1; - size_t stride = 1; - float temperature = 1.0f; - float top_p = 1.0f; - float min_p = 0.15f; - float repetition_penalty = 1.1f; - size_t top_k = 0; - size_t random_seed = 0; - - size_t index_value = 0; - size_t num_classes = 0; - size_t num_groups = 0; - size_t dst_height = 0; - size_t dst_width = 0; - bool align_corners = true; - bool normalize_routing = false; - size_t num_experts = 0; - size_t num_experts_per_tok = 0; - bool moe_gated = true; - Activation activation = Activation::SILU; - - std::vector bias_values; - std::vector bias_indices; - - const int8_t* cached_keys_int8 = nullptr; - const int8_t* cached_values_int8 = nullptr; - const float* cached_k_scales = nullptr; - const float* cached_v_scales = nullptr; - size_t cache_seq_len = 0; - size_t num_kv_heads = 0; - size_t head_dim = 0; - size_t num_fft_bins = 0; - size_t chunk_size = 0; - size_t num_altup_inputs = 0; - size_t v_head_dim = 0; - size_t kernel_size = 0; -}; - -struct GraphNode { - size_t id; - OpType op_type; - std::vector input_ids; - BufferDesc output_buffer; - OpParams params; - - GraphNode(size_t node_id, OpType type); -}; - -using nodes_vector = std::vector>; -using node_index_map_t = std::unordered_map; - -inline const BufferDesc& get_input(const GraphNode& node, size_t idx, - const nodes_vector& nodes, - const node_index_map_t& node_index_map) { - return nodes[node_index_map.at(node.input_ids[idx])]->output_buffer; -} - -struct AxisDims { - size_t outer, axis_size, inner; - static AxisDims from_shape(const std::vector& shape, size_t axis) { - AxisDims d; - d.outer = 1; - for (size_t i = 0; i < axis; i++) d.outer *= shape[i]; - d.axis_size = shape[axis]; - d.inner = 1; - for (size_t i = axis + 1; i < shape.size(); i++) d.inner *= shape[i]; - return d; - } -}; - -template -void dispatch_binary_op(OpType op, const T* lhs, const T* rhs, T* output, size_t count); - -template -void dispatch_unary_op(OpType op, const T* input, T* output, size_t count, float param = 0.0f); - -void compute_node_optimized(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_dense_mlp_int4_fused_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); -void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map); - -void shrink_thread_local_buffers(); -class BufferPool { -public: - BufferPool() = default; - ~BufferPool() = default; - - BufferPool(const BufferPool&) = delete; - BufferPool& operator=(const BufferPool&) = delete; - BufferPool(BufferPool&&) noexcept = default; - BufferPool& operator=(BufferPool&&) noexcept = default; - - char* acquire(size_t byte_size); - void release(char* ptr, size_t byte_size); - void clear(); - - size_t active_bytes() const { return active_bytes_; } - size_t pool_bytes() const { return pool_bytes_; } - size_t peak_bytes() const { return peak_bytes_; } - -private: - std::unordered_map>> free_buffers_; - size_t active_bytes_ = 0; - size_t pool_bytes_ = 0; - size_t peak_bytes_ = 0; - - size_t round_up_size(size_t size) const; -}; - -namespace ValidationUtils { - void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name); - void validate_precision(Precision actual, Precision required, const std::string& op_name); - void validate_input_count(size_t actual, size_t required, const std::string& op_name); -} - - -class CactusGraph { -public: - CactusGraph(); - ~CactusGraph() = default; - - CactusGraph(const CactusGraph&) = delete; - CactusGraph& operator=(const CactusGraph&) = delete; - CactusGraph(CactusGraph&&) noexcept = default; - CactusGraph& operator=(CactusGraph&&) noexcept = default; - - struct DebugNodeEntry { - uint32_t layer_idx; - std::string name; - size_t node_id; - }; - - void save(const std::string& path); - static CactusGraph load(const std::string& path); - - size_t input(const std::vector& shape, Precision precision = Precision::INT8); - size_t precision_cast(size_t input, Precision target_precision); - size_t quantize_activations(size_t input); - - size_t add(size_t input1, size_t input2); - size_t add_clipped(size_t input1, size_t input2); - size_t subtract(size_t input1, size_t input2); - size_t multiply(size_t input1, size_t input2); - size_t divide(size_t input1, size_t input2); - - size_t scalar_add(size_t input, float value); - size_t scalar_subtract(size_t input, float value); - size_t scalar_multiply(size_t input, float value); - size_t scalar_divide(size_t input, float value); - size_t scalar_exp(size_t input); - size_t scalar_sqrt(size_t input); - size_t scalar_cos(size_t input); - size_t scalar_sin(size_t input); - size_t scalar_log(size_t input); - - size_t relu(size_t input); - size_t silu(size_t input); - size_t gelu(size_t input); - size_t gelu_erf(size_t input); - size_t sigmoid(size_t input); - size_t tanh(size_t input); - size_t glu(size_t input, int axis = -1); - - size_t abs(size_t input); - size_t pow(size_t input, float exponent); - size_t view(size_t input, const std::vector& new_shape); - size_t flatten(size_t input, int start_dim = 0, int end_dim = -1); - - size_t matmul(size_t input1, size_t input2, bool pretransposed_rhs = false, ComputeBackend backend = ComputeBackend::CPU); - size_t transpose(size_t input, ComputeBackend backend = ComputeBackend::CPU); - size_t transposeN(size_t input, const std::vector& permutation, ComputeBackend backend = ComputeBackend::CPU); - size_t reshape(size_t input, const std::vector& new_shape); - size_t slice(size_t input, int axis, size_t start, size_t length); - size_t index(size_t input, size_t index_value, int dim); - - size_t sum(size_t input, int axis); - size_t mean(size_t input, int axis); - size_t variance(size_t input, int axis); - size_t min(size_t input, int axis); - size_t max(size_t input, int axis); - - size_t gather(size_t embeddings, size_t indices); - size_t mmap_embeddings(const std::string& filename); - size_t mmap_weights(const std::string& filename); - void set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr); - void set_interleaved(size_t node_id, bool interleaved, size_t original_N); - - void release_weight_pages(size_t node_id); - void prefetch_weight_pages(size_t node_id); - void release_all_weight_pages(); - size_t embedding(const std::string& filename, size_t indices); - size_t embedding(size_t embedding_tensor, size_t indices); - size_t bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners = true); - - size_t layernorm(size_t input, size_t weight, size_t bias, float epsilon = 1e-5f); - size_t layernorm(size_t input, size_t weight, float epsilon = 1e-5f); // No bias version - size_t groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups = 32, float epsilon = 1e-5f); - size_t batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis = 1, float epsilon = 1e-5f); - size_t topk(size_t input, size_t k); - size_t dense_mlp_int4_fused(size_t hidden, size_t gate_weight, - size_t up_weight, size_t down_weight); - size_t moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w3_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation = Activation::SILU, - size_t per_expert_scale = 0); - size_t moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation); - size_t rms_norm(size_t input, size_t weight, float epsilon = 1e-5f); - size_t rope(size_t input, float theta, size_t position_offset = 0, ComputeBackend backend = ComputeBackend::CPU); - size_t rope_gptj(size_t input, float theta, size_t position_offset = 0, size_t rot_dim = 0, ComputeBackend backend = ComputeBackend::CPU); - size_t softmax(size_t input, int axis = -1); - size_t attention(size_t query, size_t key, size_t value, float scale, bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU); - size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend = ComputeBackend::CPU); - size_t attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend = ComputeBackend::CPU); - size_t attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, - bool is_causal = true, ComputeBackend backend = ComputeBackend::CPU, - bool additive_mask = false, size_t position_offset = 0, size_t window_size = 0, - float logit_cap = 0.0f); - size_t rel_pos_bias(size_t query, size_t relative_key, float scale); - - size_t attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, - const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, - size_t window_size = 0, size_t v_head_dim = 0); - - size_t conv1d_causal(size_t input, size_t weight, size_t kernel_size, size_t dilation = 1); - size_t conv1d_k3(size_t input, size_t weight, size_t stride); - size_t conv1d_k7s3(size_t input, size_t weight, size_t bias); - size_t conv1d(size_t input, size_t weight, size_t stride); - size_t conv1d(size_t input, size_t weight, size_t bias, size_t stride); - size_t conv1d_same_depthwise_k9(size_t input, size_t weight); - size_t conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias); - size_t conv1d_pointwise(size_t input, size_t weight); - size_t conv1d_pointwise(size_t input, size_t weight, size_t bias); - size_t conv2d_k3s2p1(size_t input, size_t weight); - size_t conv2d_k3s2p1(size_t input, size_t weight, size_t bias); - size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight); - size_t conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias); - size_t conv2d_pointwise_1x1(size_t input, size_t weight); - size_t conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias); - - size_t lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh); - size_t gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, float scale = 0.0f); - size_t gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, size_t chunk_size = 64, float scale = 0.0f); - size_t stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins); - - size_t altup_predict(size_t coefs, const size_t* streams, size_t num_streams); - size_t altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions); - - size_t gaussian_topk(size_t input, float ppf); - - size_t maxpool1d(size_t input, size_t kernel_size, size_t stride); - size_t leaky_relu(size_t input, float negative_slope = 0.01f); - size_t bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, - size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd); - size_t conv2d_k3s1p1(size_t input, size_t weight); - size_t conv2d_k3s1p1(size_t input, size_t weight, size_t bias); - size_t stats_pool(size_t input); - size_t weighted_stats_pool(size_t input, size_t weights); - - size_t sample(size_t logits, float temperature = 0.6f, float top_p = 0.95f, size_t top_k = 20, - const std::unordered_map& logit_bias = {}); - size_t sample_with_options(size_t logits, float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, const std::unordered_map& logit_bias = {}); - - size_t concat(size_t input1, size_t input2, int axis = 0); - size_t cat(const std::vector& inputs, int axis); - size_t scatter_topk(size_t indices, size_t values, size_t num_classes); - - void set_input(size_t node_id, const void* data, Precision precision); - void set_external_input(size_t node_id, void* data, Precision precision); - void* get_output(size_t node_id); - - void execute(const std::string& profile_file = ""); - void hard_reset(); - void soft_reset(); - void soft_reset_keep_pool(); - void set_prefill_mode(bool enabled) { prefill_mode_ = enabled; } - - void register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); - void capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id); - const std::vector& get_debug_nodes() const; - void clear_debug_nodes(); - - size_t add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params = {}); - const BufferDesc& get_output_buffer(size_t node_id) const; - void allocate_buffers(); - size_t get_node_count() const; - - size_t persistent(size_t source_node); - bool is_populated(size_t persistent_node_id) const; - void invalidate_persistent(size_t persistent_node_id); - - std::vector> nodes_; - std::unordered_map node_index_map_; - -private: - static CactusGraph from_serialized(const GraphFile::SerializedGraph& serialized); - size_t next_node_id_; - std::vector> mapped_files_; - std::unordered_map weight_cache_; - std::unordered_map node_to_mapped_file_; - std::vector debug_nodes_; - BufferPool buffer_pool_; - bool prefill_mode_ = false; - - std::unordered_set persistent_node_ids_; - std::unordered_set populated_node_ids_; -}; - - -namespace GraphFile { - struct LoadedNode { - size_t node_id; - std::vector shape; - Precision precision; - size_t byte_size; - }; - - struct GraphHeader { - uint32_t magic; - uint32_t version; - uint32_t node_count; - uint32_t flags = 0; - }; - - struct NodeEntry { - uint32_t index; // serialized node index 0..n-1 - OpType op_type; - std::vector inputs; - std::vector output_shape; - Precision precision; - OpParams params; - }; - - struct SerializedGraph { - GraphHeader header; - std::vector nodes; - std::vector graph_inputs; // IDs of serialized inputs - std::vector graph_outputs; // IDs of serialized outputs - }; - - SerializedGraph load_graph(const std::string& filename); - void save_graph(const CactusGraph& graph, const std::string& filename); - - void save_node(CactusGraph& graph, size_t node_id, const std::string& filename); - - class MappedFile { - public: - MappedFile(const std::string& filename); - ~MappedFile(); - - MappedFile(const MappedFile&) = delete; - MappedFile& operator=(const MappedFile&) = delete; - MappedFile(MappedFile&& other) noexcept; - MappedFile& operator=(MappedFile&& other) noexcept; - - const std::vector& shape() const; - Precision precision() const; - size_t byte_size() const; - - size_t group_size() const { return group_size_; } - size_t num_groups() const { return num_groups_; } - const void* scales_data() const; - - bool is_interleaved() const { return is_interleaved_; } - size_t original_N() const { return original_N_; } - - void* data(); - const void* data() const; - - template - const T* typed_data() const; - - void release_pages(); - void prefetch_pages(); - - private: - int fd_; - void* mapped_data_; - size_t file_size_, data_offset_; - std::vector shape_; - Precision precision_; - size_t byte_size_; - size_t group_size_ = 0; - size_t num_groups_ = 0; - size_t scales_offset_ = 0; - size_t scales_bytes_ = 0; - uint32_t alignment_ = 32; - - bool is_interleaved_ = false; - size_t original_N_ = 0; - - void parse_header(); - void apply_madvise_hints(); - }; -} - -#endif diff --git a/cactus/graph/graph_builder.cpp b/cactus/graph/graph_builder.cpp deleted file mode 100644 index ca7b57edc..000000000 --- a/cactus/graph/graph_builder.cpp +++ /dev/null @@ -1,1707 +0,0 @@ -#include "graph.h" -#include -#include -#include -#include - -size_t CactusGraph::input(const std::vector& shape, Precision precision) { - return add_node(OpType::INPUT, {}, shape, {.output_precision = precision}); -} - -size_t CactusGraph::add(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::ADD, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::add_clipped(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::ADD_CLIPPED, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::subtract(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::SUBTRACT, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::multiply(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::MULTIPLY, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::divide(size_t input1, size_t input2) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - BroadcastInfo broadcast_info = BroadcastInfo::compute(lhs_buffer.shape, rhs_buffer.shape); - OpParams params{.broadcast_info = broadcast_info}; - - return add_node(OpType::DIVIDE, {input1, input2}, broadcast_info.output_shape, params); -} - -size_t CactusGraph::abs(size_t input) { - const auto& input_buffer = get_output_buffer(input); - OpParams params{.output_precision = input_buffer.precision}; - return add_node(OpType::ABS, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::pow(size_t input, float exponent) { - const auto& input_buffer = get_output_buffer(input); - OpParams params{.scalar = exponent, .output_precision = input_buffer.precision}; - return add_node(OpType::POW, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::view(size_t input, const std::vector& new_shape) { - const auto& input_buffer = get_output_buffer(input); - - size_t input_elements = 1; - for (size_t dim : input_buffer.shape) { - input_elements *= dim; - } - - size_t new_elements = 1; - for (size_t dim : new_shape) { - new_elements *= dim; - } - - if (input_elements != new_elements) { - throw std::runtime_error("View operation requires total number of elements to remain the same"); - } - - OpParams params{.new_shape = new_shape}; - return add_node(OpType::VIEW, {input}, new_shape, params); -} - -size_t CactusGraph::flatten(size_t input, int start_dim, int end_dim) { - const auto& input_buffer = get_output_buffer(input); - const auto& shape = input_buffer.shape; - size_t rank = shape.size(); - - if (start_dim < 0) start_dim += rank; - if (end_dim < 0) end_dim += rank; - - if (start_dim < 0 || static_cast(start_dim) >= rank || - end_dim < 0 || static_cast(end_dim) >= rank || - start_dim > end_dim) { - throw std::runtime_error("Invalid start_dim or end_dim for flatten operation"); - } - - std::vector output_shape; - - for (int i = 0; i < start_dim; ++i) { - output_shape.push_back(shape[i]); - } - - size_t flattened_dim = 1; - for (int i = start_dim; i <= end_dim; ++i) { - flattened_dim *= shape[i]; - } - output_shape.push_back(flattened_dim); - - for (size_t i = end_dim + 1; i < rank; ++i) { - output_shape.push_back(shape[i]); - } - - OpParams params{.new_shape = output_shape}; - return add_node(OpType::FLATTEN, {input}, output_shape, params); -} - -size_t CactusGraph::matmul(size_t input1, size_t input2, bool pretransposed_rhs, ComputeBackend backend) { - const auto& lhs_buffer = get_output_buffer(input1); - const auto& rhs_buffer = get_output_buffer(input2); - - if (lhs_buffer.shape.size() != 2 || rhs_buffer.shape.size() != 2) { - throw std::invalid_argument("Matrix multiplication requires 2D tensors"); - } - - size_t M = lhs_buffer.shape[0]; - size_t K = lhs_buffer.shape[1]; - size_t rhs_K = pretransposed_rhs ? rhs_buffer.shape[1] : rhs_buffer.shape[0]; - - // For interleaved weights, use original_N (shape stores padded N) - size_t N; - if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { - N = rhs_buffer.original_N; - } else { - N = pretransposed_rhs ? rhs_buffer.shape[0] : rhs_buffer.shape[1]; - } - - if (K != rhs_K) { - std::cout << "Matrix dimensions incompatible for multiplication: " - << "lhs=[" << M << "," << K << "] rhs=[" << rhs_buffer.shape[0] << "," << rhs_buffer.shape[1] << "]" - << " pretransposed=" << pretransposed_rhs - << " (K=" << K << " != rhs_K=" << rhs_K << ")" << std::endl; - throw std::invalid_argument("Matrix dimensions incompatible for multiplication"); - } - - std::vector output_shape = {M, N}; - OpParams params{.pretransposed_rhs = pretransposed_rhs, .backend = backend}; - return add_node(OpType::MATMUL, {input1, input2}, output_shape, params); -} - -size_t CactusGraph::transpose(size_t input, ComputeBackend backend) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape = input_buffer.shape; - - if (output_shape.size() >= 2) { - std::swap(output_shape[output_shape.size()-2], output_shape[output_shape.size()-1]); - } - - std::vector permutation; - for (size_t i = 0; i < input_buffer.shape.size(); ++i) { - permutation.push_back(i); - } - if (permutation.size() >= 2) { - std::swap(permutation[permutation.size()-2], permutation[permutation.size()-1]); - } - - OpParams params{.permutation = permutation, .backend = backend}; - return add_node(OpType::TRANSPOSE, {input}, output_shape, params); -} - -size_t CactusGraph::transposeN(size_t input, const std::vector& permutation, ComputeBackend backend) { - const auto& input_buffer = get_output_buffer(input); - if (permutation.size() != input_buffer.shape.size()) { - throw std::runtime_error("transposeN permutation size must match tensor rank"); - } - std::vector output_shape(permutation.size()); - for (size_t i = 0; i < permutation.size(); ++i) { - size_t p = permutation[i]; - if (p >= input_buffer.shape.size()) { - throw std::runtime_error("transposeN permutation index out of range"); - } - output_shape[i] = input_buffer.shape[p]; - } - OpParams params{.permutation = permutation, .backend = backend}; - return add_node(OpType::TRANSPOSE, {input}, output_shape, params); -} - -size_t CactusGraph::reshape(size_t input, const std::vector& new_shape) { - OpParams params{.new_shape = new_shape}; - return add_node(OpType::RESHAPE, {input}, new_shape, params); -} - -size_t CactusGraph::index(size_t input, size_t index_value, int dim) { - const auto& input_buffer = get_output_buffer(input); - const auto& shape = input_buffer.shape; - - if (shape.empty()) { - throw std::invalid_argument("Cannot index a scalar tensor"); - } - - int actual_dim = dim; - if (actual_dim < 0) { - actual_dim += static_cast(shape.size()); - } - - if (actual_dim < 0 || static_cast(actual_dim) >= shape.size()) { - throw std::invalid_argument("Index dimension out of bounds"); - } - - if (index_value >= shape[actual_dim]) { - throw std::invalid_argument("Index value " + std::to_string(index_value) + - " out of bounds for dimension " + std::to_string(actual_dim) + - " with size " + std::to_string(shape[actual_dim])); - } - - std::vector output_shape; - for (size_t i = 0; i < shape.size(); ++i) { - if (static_cast(i) != actual_dim) { - output_shape.push_back(shape[i]); - } - } - - if (output_shape.empty()) { - output_shape = {1}; - } - - OpParams params{.axis = actual_dim, .output_precision = input_buffer.precision, .index_value = index_value}; - return add_node(OpType::INDEX, {input}, output_shape, params); -} - -size_t CactusGraph::sum(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape; - - if (axis == -1) { - output_shape = {1}; - } else { - output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - } - - OpParams params{.axis = axis, .output_precision = input_buffer.precision}; - return add_node(OpType::SUM, {input}, output_shape, params); -} - -size_t CactusGraph::mean(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape; - - if (axis == -1) { - output_shape = {1}; - } else { - output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - } - - OpParams params{.axis = axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MEAN, {input}, output_shape, params); -} - -size_t CactusGraph::variance(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape; - - if (axis == -1) { - output_shape = {1}; - } else { - output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - } - - OpParams params{.axis = axis, .output_precision = input_buffer.precision}; - return add_node(OpType::VARIANCE, {input}, output_shape, params); -} - -size_t CactusGraph::min(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape; - - if (axis == -1) { - output_shape = {1}; - } else { - output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - } - - OpParams params{.axis = axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MIN, {input}, output_shape, params); -} - -size_t CactusGraph::max(size_t input, int axis) { - const auto& input_buffer = get_output_buffer(input); - std::vector output_shape; - - if (axis == -1) { - output_shape = {1}; - } else { - output_shape = input_buffer.shape; - output_shape.erase(output_shape.begin() + axis); - if (output_shape.empty()) { - output_shape = {1}; - } - } - - OpParams params{.axis = axis, .output_precision = input_buffer.precision}; - return add_node(OpType::MAX, {input}, output_shape, params); -} - -size_t CactusGraph::rms_norm(size_t input, size_t weight, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::RMS_NORM, {input, weight}, {}, params); -} - -size_t CactusGraph::rope(size_t input, float theta, size_t position_offset, ComputeBackend backend) { - OpParams params{.theta = theta, .position_offset = position_offset, .backend = backend}; - return add_node(OpType::ROPE, {input}, {}, params); -} - -size_t CactusGraph::softmax(size_t input, int axis) { - OpParams params{.axis = axis}; - return add_node(OpType::SOFTMAX, {input}, {}, params); -} - -size_t CactusGraph::topk(size_t input, size_t k) { - const auto& input_buffer = get_output_buffer(input); - - if (input_buffer.shape.empty()) { - throw std::runtime_error("TopK requires non-empty input tensor"); - } - - std::vector output_shape = {2, input_buffer.shape[0], k}; - OpParams params{.output_precision = Precision::FP32, .top_k = k}; - - return add_node(OpType::TOPK, {input}, output_shape, params); -} - -size_t CactusGraph::dense_mlp_int4_fused(size_t hidden, size_t gate_weight, - size_t up_weight, size_t down_weight) { - const auto& hidden_buf = get_output_buffer(hidden); - if (hidden_buf.shape.size() < 2) { - throw std::runtime_error("dense_mlp_int4_fused requires hidden of rank >= 2"); - } - const auto& down_buf = get_output_buffer(down_weight); - size_t hidden_dim; - if (down_buf.is_interleaved && down_buf.original_N > 0) { - hidden_dim = down_buf.original_N; - } else { - hidden_dim = down_buf.shape[down_buf.shape.size() - 2]; - } - - std::vector output_shape = hidden_buf.shape; - output_shape[output_shape.size() - 1] = hidden_dim; - - OpParams params; - params.output_precision = Precision::FP16; - - return add_node(OpType::DENSE_MLP_INT4_FUSED, - {hidden, gate_weight, up_weight, down_weight}, - output_shape, params); -} - -size_t CactusGraph::moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w3_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation, - size_t per_expert_scale) { - const auto& hidden_buffer = get_output_buffer(hidden); - const auto& routing_buffer = get_output_buffer(routing_probs); - const auto& topk_buffer = get_output_buffer(topk_indices); - - if (hidden_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); - } - if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); - } - if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { - throw std::runtime_error("moe_layer token dimension mismatch across inputs"); - } - if (w1_weights.size() != num_experts || w3_weights.size() != num_experts || w2_weights.size() != num_experts) { - throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w3, w2"); - } - - std::vector input_ids; - input_ids.reserve(3 + 3 * num_experts + 1); - input_ids.push_back(hidden); - input_ids.push_back(routing_probs); - input_ids.push_back(topk_indices); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w3_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); - if (per_expert_scale != 0) input_ids.push_back(per_expert_scale); - - OpParams params; - params.num_experts = num_experts; - params.num_experts_per_tok = num_experts_per_tok; - params.normalize_routing = normalize_routing; - params.epsilon = epsilon; - params.scalar = routed_scaling_factor; - params.output_precision = hidden_buffer.precision; - params.activation = activation; - params.moe_gated = true; - - return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); -} - -size_t CactusGraph::moe_layer(size_t hidden, - size_t routing_probs, - size_t topk_indices, - const std::vector& w1_weights, - const std::vector& w2_weights, - size_t num_experts, - size_t num_experts_per_tok, - bool normalize_routing, - float epsilon, - float routed_scaling_factor, - Activation activation) { - const auto& hidden_buffer = get_output_buffer(hidden); - const auto& routing_buffer = get_output_buffer(routing_probs); - const auto& topk_buffer = get_output_buffer(topk_indices); - - if (hidden_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects [tokens, hidden_dim] for hidden"); - } - if (routing_buffer.shape.size() != 2 || topk_buffer.shape.size() != 2) { - throw std::runtime_error("moe_layer expects 2D routing_probs and topk_indices"); - } - if (routing_buffer.shape[0] != hidden_buffer.shape[0] || topk_buffer.shape[0] != hidden_buffer.shape[0]) { - throw std::runtime_error("moe_layer token dimension mismatch across inputs"); - } - if (w1_weights.size() != num_experts || w2_weights.size() != num_experts) { - throw std::runtime_error("moe_layer expects num_experts weight tensors for each of w1, w2"); - } - - std::vector input_ids; - input_ids.reserve(3 + 2 * num_experts); - input_ids.push_back(hidden); - input_ids.push_back(routing_probs); - input_ids.push_back(topk_indices); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w1_weights[i]); - for (size_t i = 0; i < num_experts; ++i) input_ids.push_back(w2_weights[i]); - - OpParams params; - params.num_experts = num_experts; - params.num_experts_per_tok = num_experts_per_tok; - params.normalize_routing = normalize_routing; - params.epsilon = epsilon; - params.scalar = routed_scaling_factor; - params.output_precision = hidden_buffer.precision; - params.moe_gated = false; - params.activation = activation; - - return add_node(OpType::MOE_LAYER, input_ids, hidden_buffer.shape, params); -} - -size_t CactusGraph::layernorm(size_t input, size_t weight, size_t bias, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::LAYERNORM, {input, weight, bias}, {}, params); -} - -size_t CactusGraph::layernorm(size_t input, size_t weight, float epsilon) { - OpParams params{.epsilon = epsilon}; - return add_node(OpType::LAYERNORM, {input, weight}, {}, params); -} - -size_t CactusGraph::groupnorm(size_t input, size_t weight, size_t bias, size_t num_groups, float epsilon) { - OpParams params{.epsilon = epsilon, .num_groups = num_groups}; - return add_node(OpType::GROUPNORM, {input, weight, bias}, {}, params); -} - -size_t CactusGraph::batchnorm(size_t input, size_t weight, size_t bias, size_t running_mean, size_t running_var, int axis, float epsilon) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - const auto& rm = get_output_buffer(running_mean); - const auto& rv = get_output_buffer(running_var); - - if (xin.shape.empty()) { - throw std::runtime_error("batchnorm expects non-scalar input"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(xin.shape.size()); - } - if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { - throw std::runtime_error("batchnorm axis out of range"); - } - - const size_t C = xin.shape[static_cast(actual_axis)]; - if (w.total_size != C || b.total_size != C || rm.total_size != C || rv.total_size != C) { - throw std::runtime_error("batchnorm parameter size mismatch"); - } - - OpParams params{.epsilon = epsilon, .axis = actual_axis}; - params.output_precision = xin.precision; - return add_node(OpType::BATCHNORM, {input, weight, bias, running_mean, running_var}, xin.shape, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, bool is_causal, ComputeBackend backend) { - OpParams params{.scale = scale, .is_causal = is_causal, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, ComputeBackend backend) { - OpParams params{.scale = scale, .position_offset = position_offset, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention(size_t query, size_t key, size_t value, float scale, size_t position_offset, size_t window_size, ComputeBackend backend) { - OpParams params{.scale = scale, .position_offset = position_offset, .window_size = window_size, .backend = backend}; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::attention_masked(size_t query, size_t key, size_t value, size_t mask, float scale, - bool is_causal, ComputeBackend backend, bool additive_mask, - size_t position_offset, size_t window_size, float logit_cap) { - OpParams params{ - .scale = scale, - .position_offset = position_offset, - .window_size = window_size, - .is_causal = is_causal, - .backend = backend - }; - params.attention_mask_is_additive = additive_mask; - params.logit_cap = logit_cap; - const auto& qs = get_output_buffer(query).shape; - const auto& vs = get_output_buffer(value).shape; - return add_node(OpType::ATTENTION, {query, key, value, mask}, {qs[0], qs[1], qs[2], vs[3]}, params); -} - -size_t CactusGraph::rel_pos_bias(size_t query, size_t relative_key, float scale) { - const auto& q = get_output_buffer(query); - const auto& r = get_output_buffer(relative_key); - - if (q.shape.size() != 4) { - throw std::runtime_error("rel_pos_bias expects query [B, T, H, D]"); - } - if (r.shape.size() != 4) { - throw std::runtime_error("rel_pos_bias expects relative_key [B, R, H, D]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t H = q.shape[2]; - const size_t D = q.shape[3]; - const size_t R = r.shape[1]; - - if (r.shape[0] != 1 && r.shape[0] != B) { - throw std::runtime_error("rel_pos_bias relative_key batch must be 1 or match query batch"); - } - if (r.shape[2] != H || r.shape[3] != D) { - throw std::runtime_error("rel_pos_bias expects matching [H, D] between query and relative_key"); - } - if (R < (2 * T - 1)) { - throw std::runtime_error("rel_pos_bias requires relative_key length >= 2*T-1"); - } - - OpParams params{.scale = scale, .output_precision = q.precision}; - return add_node(OpType::REL_POS_BIAS, {query, relative_key}, {B, H, T, T}, params); -} - -size_t CactusGraph::attention_int8_hybrid(size_t query, size_t key_new, size_t value_new, float scale, size_t position_offset, - const int8_t* cached_keys, const int8_t* cached_values, - const float* k_scales, const float* v_scales, - size_t cache_len, size_t num_kv_heads, size_t head_dim, - size_t window_size, size_t v_head_dim) { - OpParams params; - params.scale = scale; - params.position_offset = position_offset; - params.window_size = window_size; - params.cached_keys_int8 = cached_keys; - params.cached_values_int8 = cached_values; - params.cached_k_scales = k_scales; - params.cached_v_scales = v_scales; - params.cache_seq_len = cache_len; - params.num_kv_heads = num_kv_heads; - params.head_dim = head_dim; - params.v_head_dim = v_head_dim; - std::vector out_shape; - if (v_head_dim != 0 && v_head_dim != head_dim) { - const auto& q_buf = get_output_buffer(query); - out_shape = {q_buf.shape[0], q_buf.shape[1], q_buf.shape[2], v_head_dim}; - } - return add_node(OpType::ATTENTION_INT8_HYBRID, {query, key_new, value_new}, out_shape, params); -} - -size_t CactusGraph::conv1d_causal(size_t input, size_t weight, size_t, size_t dilation) { - OpParams params{.dilation = dilation}; - return add_node(OpType::CONV1D_CAUSAL, {input, weight}, {}, params); -} - -size_t CactusGraph::conv1d_k3(size_t input, size_t weight, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k3 expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_out,C_in,3]"); - if (w.shape[1] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k3"); - if (w.shape[2] != 3) throw std::runtime_error("K=3 expected in conv1d_k3"); - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[2]; - const size_t C_out= w.shape[0]; - const size_t K = w.shape[2]; - - const size_t pad = 1; - const size_t L_out = (L + 2 * pad - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.output_precision = xin.precision; - - std::vector out_shape{N, C_out, L_out}; - return add_node(OpType::CONV1D_K3, {input, weight}, out_shape, params); -} - -size_t CactusGraph::conv1d_k7s3(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d_k7s3 expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("weight must be [C_in, 7, C_out]"); - if (w.shape[0] != xin.shape[1]) throw std::runtime_error("C_in mismatch in conv1d_k7s3"); - if (w.shape[1] != 7) throw std::runtime_error("K=7 expected in conv1d_k7s3"); - - size_t C_out = w.shape[2]; - if (b.total_size != C_out) throw std::runtime_error("Bias size mismatch"); - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[2]; - const size_t K = 7; - const size_t stride = 3; - - const size_t L_out = (L < K) ? 0 : (L - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.output_precision = xin.precision; - - std::vector out_shape{N, C_out, L_out}; - return add_node(OpType::CONV1D_K7S3, {input, weight, bias}, out_shape, params); -} - -size_t CactusGraph::conv1d(size_t input, size_t weight, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t C_out = w.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - - OpParams params{.stride = stride}; - return add_node(OpType::CONV1D, {input, weight}, {N, C_out, L_out}, params); -} - -size_t CactusGraph::conv1d(size_t input, size_t weight, size_t bias, size_t stride) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) throw std::runtime_error("conv1d expects N,C,L"); - if (w.shape.size() != 3) throw std::runtime_error("conv1d weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t C_out = w.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - if (b.total_size != C_out) throw std::runtime_error("conv1d bias size mismatch"); - - OpParams params{.output_precision = xin.precision, .stride = stride}; - return add_node(OpType::CONV1D, {input, weight, bias}, {N, C_out, L_out}, params); -} - -size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C = xin.shape[2]; - - if (w.shape.size() == 2) { - if (w.shape[0] != C || w.shape[1] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight}, {N, L, C}, params); -} - -size_t CactusGraph::conv1d_same_depthwise_k9(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C = xin.shape[2]; - - if (w.shape.size() == 2) { - if (w.shape[0] != C || w.shape[1] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 9) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - if (b.total_size != C) { - throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_SAME_DEPTHWISE_K9, {input, weight, bias}, {N, L, C}, params); -} - -size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - - const size_t N = xin.shape[0]; - const size_t L = xin.shape[1]; - const size_t C_in = xin.shape[2]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 3) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_POINTWISE, {input, weight}, {N, L, C_out}, params); -} - -size_t CactusGraph::conv1d_pointwise(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - - const size_t C_in = xin.shape[2]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 3) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - if (b.total_size != C_out) { - throw std::runtime_error("conv1d_pointwise bias size mismatch"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV1D_POINTWISE, {input, weight, bias}, {xin.shape[0], xin.shape[1], C_out}, params); -} - -size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - if (w.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s2p1 weight must match [C_out, C_in, 3, 3]"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S2P1, {input, weight}, {N, C_out, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_k3s2p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s2p1 weight must be [C_out, C_in, 3, 3]"); - } - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_k3s2p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t N = xin.shape[0]; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S2P1, {input, weight, bias}, {N, C_out, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (w.shape.size() == 4) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight}, {N, C, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_depthwise_k3s2p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - - const size_t C = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() == 3) { - if (w.shape[0] != C || w.shape[1] != 3 || w.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (w.shape.size() == 4) { - if (w.shape[0] != C || w.shape[1] != 1 || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - if (b.total_size != C) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - const size_t N = xin.shape[0]; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_DEPTHWISE_K3S2P1, {input, weight, bias}, {N, C, H_out, W_out}, params); -} - -size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 4) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::conv2d_pointwise_1x1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - size_t C_out = 0; - - if (w.shape.size() == 2) { - C_out = w.shape[0]; - if (w.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (w.shape.size() == 4) { - C_out = w.shape[0]; - if (w.shape[1] != C_in || w.shape[2] != 1 || w.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_POINTWISE_1X1, {input, weight, bias}, {xin.shape[0], C_out, H, W}, params); -} - -size_t CactusGraph::lstm_cell(size_t input, size_t h_prev, size_t c_prev, size_t weight_ih, size_t weight_hh, size_t bias_ih, size_t bias_hh) { - const auto& h_buffer = get_output_buffer(h_prev); - std::vector output_shape = {h_buffer.shape[0], h_buffer.shape[1], 2}; - return add_node(OpType::LSTM_CELL, {input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh}, output_shape, {}); -} - -size_t CactusGraph::gated_deltanet_decode(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, float scale) { - const auto& q = get_output_buffer(query); - const auto& k = get_output_buffer(key); - const auto& v = get_output_buffer(value); - const auto& g = get_output_buffer(gate_log); - const auto& b = get_output_buffer(beta); - const auto& s = get_output_buffer(initial_state); - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_decode expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("gated_deltanet_decode expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_decode expects initial_state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - if (T != 1) { - throw std::runtime_error("gated_deltanet_decode expects sequence length T=1"); - } - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("gated_deltanet_decode requires FP16/FP32 inputs"); - } - - float op_scale = scale; - if (op_scale == 0.0f) { - op_scale = 1.0f / std::sqrt(static_cast(K)); - } - - OpParams params; - params.scale = op_scale; - params.num_kv_heads = Hq; - params.output_precision = Precision::FP16; - return add_node(OpType::GATED_DELTANET_DECODE, - {query, key, value, gate_log, beta, initial_state}, - {B, static_cast(1 + K), Hv, V}, - params); -} - -size_t CactusGraph::gated_deltanet_prefill(size_t query, size_t key, size_t value, size_t gate_log, size_t beta, - size_t initial_state, size_t chunk_size, float scale) { - const auto& q = get_output_buffer(query); - const auto& k = get_output_buffer(key); - const auto& v = get_output_buffer(value); - const auto& g = get_output_buffer(gate_log); - const auto& b = get_output_buffer(beta); - const auto& s = get_output_buffer(initial_state); - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_prefill expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("gated_deltanet_prefill expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("gated_deltanet_prefill expects initial_state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("gated_deltanet_prefill requires FP16/FP32 inputs"); - } - - if (chunk_size == 0) { - chunk_size = 64; - } - - float op_scale = scale; - if (op_scale == 0.0f) { - op_scale = 1.0f / std::sqrt(static_cast(K)); - } - - OpParams params; - params.scale = op_scale; - params.chunk_size = chunk_size; - params.num_kv_heads = Hq; - params.output_precision = Precision::FP16; - return add_node(OpType::GATED_DELTANET_PREFILL, - {query, key, value, gate_log, beta, initial_state}, - {B, T + K, Hv, V}, - params); -} - -size_t CactusGraph::stft(size_t input, size_t weight, size_t stride, size_t num_fft_bins) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 3) throw std::runtime_error("stft expects N,C,L input"); - if (w.shape.size() != 3) throw std::runtime_error("stft weight expects [C_out, C_in, K]"); - - size_t N = xin.shape[0]; - size_t L = xin.shape[2]; - size_t K = w.shape[2]; - size_t L_out = (L - K) / stride + 1; - - OpParams params{}; - params.stride = stride; - params.num_fft_bins = num_fft_bins; - - return add_node(OpType::STFT, {input, weight}, {N, 2 * num_fft_bins, L_out}, params); -} - -size_t CactusGraph::concat(size_t input1, size_t input2, int axis) { - const auto& buffer1 = get_output_buffer(input1); - const auto& buffer2 = get_output_buffer(input2); - - if (buffer1.shape.size() != buffer2.shape.size()) { - throw std::runtime_error("Concat requires inputs with same number of dimensions"); - } - - std::vector output_shape = buffer1.shape; - size_t ndims = output_shape.size(); - - if (axis < 0) axis += ndims; - if (axis < 0 || static_cast(axis) >= ndims) { - throw std::runtime_error("Invalid axis for concat operation"); - } - - for (size_t i = 0; i < ndims; ++i) { - if (i != static_cast(axis) && buffer1.shape[i] != buffer2.shape[i]) { - throw std::runtime_error("Concat inputs must have same shape except on concat axis"); - } - } - - output_shape[axis] = buffer1.shape[axis] + buffer2.shape[axis]; - - OpParams params; - params.axis = axis; - return add_node(OpType::CONCAT, {input1, input2}, output_shape, params); -} - -size_t CactusGraph::cat(const std::vector& inputs, int axis) { - if (inputs.empty()) { - throw std::runtime_error("Cat requires at least one input"); - } - - const auto& first_buffer = get_output_buffer(inputs[0]); - std::vector output_shape = first_buffer.shape; - size_t ndims = output_shape.size(); - - if (axis < 0) axis += ndims; - if (axis < 0 || static_cast(axis) >= ndims) { - throw std::runtime_error("Invalid axis for cat operation"); - } - - for (size_t i = 1; i < inputs.size(); ++i) { - const auto& buffer = get_output_buffer(inputs[i]); - if (buffer.shape.size() != ndims) { - throw std::runtime_error("All inputs to cat must have same number of dimensions"); - } - for (size_t d = 0; d < ndims; ++d) { - if (d != static_cast(axis) && buffer.shape[d] != output_shape[d]) { - throw std::runtime_error("All inputs to cat must have same shape except on cat axis"); - } - } - output_shape[axis] += buffer.shape[axis]; - } - - OpParams params; - params.axis = axis; - return add_node(OpType::CAT, inputs, output_shape, params); -} - -size_t CactusGraph::scatter_topk(size_t indices, size_t values, size_t num_classes) { - const auto& indices_buffer = get_output_buffer(indices); - const auto& values_buffer = get_output_buffer(values); - - if (indices_buffer.shape != values_buffer.shape) { - throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); - } - if (indices_buffer.shape.size() != 2) { - throw std::runtime_error("ScatterTopK currently supports 2D tensors [batch, top_k]"); - } - if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { - throw std::runtime_error("ScatterTopK expects FP32 indices and values"); - } - - std::vector output_shape = {num_classes, indices_buffer.shape[0]}; - OpParams params{.output_precision = Precision::FP32, .num_classes = num_classes}; - return add_node(OpType::SCATTER_TOPK, {indices, values}, output_shape, params); -} - -size_t CactusGraph::sample(size_t logits, float temperature, float top_p, size_t top_k, - const std::unordered_map& logit_bias) { - return this->sample_with_options(logits, temperature, top_p, 0.15f, 1.1f, top_k, logit_bias); -} - -size_t CactusGraph::sample_with_options(size_t logits, float temperature, float top_p, - float min_p, float repetition_penalty, size_t top_k, - const std::unordered_map& logit_bias) { - const auto& logits_buffer = get_output_buffer(logits); - - if (logits_buffer.shape.empty()) { - throw std::runtime_error("Sample requires non-empty logits tensor"); - } - - OpParams params; - params.temperature = temperature; - params.top_p = top_p; - params.min_p = min_p; - params.repetition_penalty = repetition_penalty; - params.top_k = top_k; - params.random_seed = std::chrono::high_resolution_clock::now().time_since_epoch().count(); - params.output_precision = Precision::FP32; - - if (!logit_bias.empty()) { - params.bias_indices.reserve(logit_bias.size()); - params.bias_values.reserve(logit_bias.size()); - for (const auto& [idx, val] : logit_bias) { - params.bias_indices.push_back(idx); - params.bias_values.push_back(val); - } - } - - std::vector output_shape = {1}; - return add_node(OpType::SAMPLE, {logits}, output_shape, params); -} - -size_t CactusGraph::scalar_add(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_ADD, {input}, {}, params); -} - -size_t CactusGraph::scalar_subtract(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_SUBTRACT, {input}, {}, params); -} - -size_t CactusGraph::scalar_multiply(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_MULTIPLY, {input}, {}, params); -} - -size_t CactusGraph::scalar_divide(size_t input, float value) { - OpParams params{}; - params.scalar = value; - params.output_precision = get_output_buffer(input).precision; - return add_node(OpType::SCALAR_DIVIDE, {input}, {}, params); -} - -size_t CactusGraph::scalar_exp(size_t input) { - return add_node(OpType::SCALAR_EXP, {input}, {}); -} - -size_t CactusGraph::scalar_sqrt(size_t input) { - return add_node(OpType::SCALAR_SQRT, {input}, {}); -} - -size_t CactusGraph::scalar_cos(size_t input) { - return add_node(OpType::SCALAR_COS, {input}, {}); -} - -size_t CactusGraph::scalar_sin(size_t input) { - return add_node(OpType::SCALAR_SIN, {input}, {}); -} - -size_t CactusGraph::scalar_log(size_t input) { - return add_node(OpType::SCALAR_LOG, {input}, {}); -} - -size_t CactusGraph::relu(size_t input) { - return add_node(OpType::RELU, {input}, {}); -} - -size_t CactusGraph::silu(size_t input) { - return add_node(OpType::SILU, {input}, {}); -} - -size_t CactusGraph::gelu(size_t input) { - return add_node(OpType::GELU, {input}, {}); -} - -size_t CactusGraph::gelu_erf(size_t input) { - return add_node(OpType::GELU_ERF, {input}, {}); -} - -size_t CactusGraph::sigmoid(size_t input) { - return add_node(OpType::SIGMOID, {input}, {}); -} - -size_t CactusGraph::tanh(size_t input) { - return add_node(OpType::TANH, {input}, {}); -} - -size_t CactusGraph::glu(size_t input, int axis) { - const auto& xin = get_output_buffer(input); - if (xin.shape.empty()) { - throw std::runtime_error("glu expects non-scalar input"); - } - - int actual_axis = axis; - if (actual_axis < 0) { - actual_axis += static_cast(xin.shape.size()); - } - if (actual_axis < 0 || static_cast(actual_axis) >= xin.shape.size()) { - throw std::runtime_error("glu axis out of range"); - } - - std::vector out_shape = xin.shape; - const size_t axis_size = out_shape[static_cast(actual_axis)]; - if ((axis_size % 2) != 0) { - throw std::runtime_error("glu split dimension must be even"); - } - out_shape[static_cast(actual_axis)] = axis_size / 2; - - OpParams params{}; - params.axis = actual_axis; - params.output_precision = xin.precision; - return add_node(OpType::GLU, {input}, out_shape, params); -} - -size_t CactusGraph::rope_gptj(size_t input, float theta, size_t position_offset, size_t rot_dim, ComputeBackend backend) { - OpParams params; - params.theta = theta; - params.position_offset = position_offset; - params.scalar = static_cast(rot_dim); - params.backend = backend; - return add_node(OpType::ROPE_GPTJ, {input}, {}, params); -} - -size_t CactusGraph::gather(size_t tensor, size_t indices) { - const auto& tensor_buffer = get_output_buffer(tensor); - const auto& idx_shape = get_output_buffer(indices).shape; - - if (tensor_buffer.shape.empty()) { - throw std::runtime_error("Cannot gather from scalar tensor"); - } - - std::vector output_shape = idx_shape; - for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { - output_shape.push_back(tensor_buffer.shape[i]); - } - - OpParams params; - params.output_precision = tensor_buffer.precision; - - return add_node(OpType::GATHER, {tensor, indices}, output_shape, params); -} - -size_t CactusGraph::slice(size_t input, int axis, size_t start, size_t length) { - const auto& input_buffer = get_output_buffer(input); - if (input_buffer.shape.empty()) { - throw std::runtime_error("Cannot slice a scalar tensor"); - } - - size_t axis_index = static_cast(axis); - size_t axis_size = input_buffer.shape[axis_index]; - - if (start + length > axis_size) { - throw std::runtime_error("Slice range extends beyond axis size"); - } - - std::vector output_shape = input_buffer.shape; - output_shape[axis_index] = length; - - OpParams params; - params.axis = axis_index; - params.slice_start = start; - params.slice_length = length; - params.output_precision = input_buffer.precision; - - return add_node(OpType::SLICE, {input}, output_shape, params); -} - -size_t CactusGraph::embedding(size_t embedding_tensor, size_t indices) { - const auto& emb_buffer = get_output_buffer(embedding_tensor); - const auto& idx_shape = get_output_buffer(indices).shape; - - if (emb_buffer.shape.size() != 2) { - std::cerr << "Error: Embedding tensor " << embedding_tensor << " has invalid shape: ["; - for(auto d : emb_buffer.shape) std::cerr << d << ","; - std::cerr << "]. OpType=" << (int)nodes_[node_index_map_[embedding_tensor]]->op_type - << " ExtData=" << emb_buffer.external_data << std::endl; - throw std::runtime_error("Embedding tensor must be 2D [vocab_size, hidden_dim]"); - } - - std::vector output_shape = idx_shape; - output_shape.push_back(emb_buffer.shape[1]); - - OpParams params; - params.output_precision = Precision::FP16; - return add_node(OpType::EMBEDDING, {embedding_tensor, indices}, output_shape, params); -} - -size_t CactusGraph::bilinear_interpolation(size_t pos_embeds, size_t dst_height, size_t dst_width, bool align_corners) { - const auto& pos_embeds_buffer = get_output_buffer(pos_embeds); - size_t embed_dim = pos_embeds_buffer.shape[1]; - std::vector output_shape = {dst_height * dst_width, embed_dim}; - - OpParams params; - params.dst_height = dst_height; - params.dst_width = dst_width; - params.align_corners = align_corners; - params.output_precision = Precision::FP16; - - return add_node(OpType::BILINEAR_INTERPOLATION, {pos_embeds}, output_shape, params); -} - -size_t CactusGraph::precision_cast(size_t input, Precision target_precision) { - OpParams params{}; - params.output_precision = target_precision; - return add_node(OpType::PRECISION_CAST, {input}, {}, params); -} - -size_t CactusGraph::quantize_activations(size_t input) { - const auto& input_buffer = get_output_buffer(input); - - if (input_buffer.precision != Precision::FP16) { - throw std::invalid_argument("quantize_activations requires FP16 input"); - } - - OpParams params{}; - params.output_precision = Precision::INT8; - return add_node(OpType::QUANTIZE_ACTIVATIONS, {input}, input_buffer.shape, params); -} - -size_t CactusGraph::add_node(OpType op_type, const std::vector& inputs, const std::vector& output_shape, const OpParams& params) { - auto node = std::make_unique(next_node_id_, op_type); - node->input_ids = inputs; - node->params = params; - - std::vector result_shape = output_shape; - if (result_shape.empty() && !inputs.empty()) { - result_shape = nodes_[node_index_map_[inputs[0]]]->output_buffer.shape; - } - - Precision result_precision = params.output_precision; - if (op_type == OpType::PRECISION_CAST) { - result_precision = params.output_precision; - } else if (result_precision == Precision::INT8 && !inputs.empty()) { - result_precision = nodes_[node_index_map_[inputs[0]]]->output_buffer.precision; - } - - node->output_buffer = BufferDesc(result_shape, result_precision); - - size_t node_id = next_node_id_++; - node_index_map_[node_id] = nodes_.size(); - nodes_.push_back(std::move(node)); - - return node_id; -} - -const BufferDesc& CactusGraph::get_output_buffer(size_t node_id) const { - return nodes_[node_index_map_.at(node_id)]->output_buffer; -} - -size_t CactusGraph::persistent(size_t source_node) { - const auto& source_buffer = get_output_buffer(source_node); - OpParams params; - params.output_precision = source_buffer.precision; - size_t node_id = add_node(OpType::PERSISTENT, {source_node}, source_buffer.shape, params); - persistent_node_ids_.insert(node_id); - return node_id; -} - -bool CactusGraph::is_populated(size_t persistent_node_id) const { - return populated_node_ids_.count(persistent_node_id) > 0; -} - -void CactusGraph::invalidate_persistent(size_t persistent_node_id) { - populated_node_ids_.erase(persistent_node_id); - persistent_node_ids_.erase(persistent_node_id); -} - -size_t CactusGraph::altup_predict(size_t coefs, const size_t* streams, size_t num_streams) { - const auto& stream0_buf = get_output_buffer(streams[0]); - - size_t seq_len = stream0_buf.shape[0]; - size_t hidden_dim = stream0_buf.shape[1]; - - std::vector input_ids = {coefs}; - for (size_t i = 0; i < num_streams; i++) - input_ids.push_back(streams[i]); - - OpParams params; - params.num_altup_inputs = num_streams; - return add_node(OpType::ALTUP_PREDICT, input_ids, {num_streams * seq_len, hidden_dim}, params); -} - -size_t CactusGraph::altup_correct(size_t coefs, size_t innovation, const size_t* predictions, size_t num_predictions) { - const auto& pred0_buf = get_output_buffer(predictions[0]); - - size_t seq_len = pred0_buf.shape[0]; - size_t hidden_dim = pred0_buf.shape[1]; - - std::vector input_ids = {coefs, innovation}; - for (size_t i = 0; i < num_predictions; i++) - input_ids.push_back(predictions[i]); - - OpParams params; - params.num_altup_inputs = num_predictions; - return add_node(OpType::ALTUP_CORRECT, input_ids, {num_predictions * seq_len, hidden_dim}, params); -} - -size_t CactusGraph::gaussian_topk(size_t input, float ppf) { - const auto& in_buf = get_output_buffer(input); - OpParams params; - params.scalar = ppf; - return add_node(OpType::GAUSSIAN_TOPK, {input}, in_buf.shape, params); -} - -size_t CactusGraph::leaky_relu(size_t input, float negative_slope) { - const auto& in_buf = get_output_buffer(input); - OpParams params; - params.scalar = negative_slope; - return add_node(OpType::LEAKY_RELU, {input}, in_buf.shape, params); -} - -size_t CactusGraph::bilstm_sequence(size_t input, size_t w_ih_fwd, size_t w_hh_fwd, size_t b_ih_fwd, size_t b_hh_fwd, - size_t w_ih_bwd, size_t w_hh_bwd, size_t b_ih_bwd, size_t b_hh_bwd) { - const auto& in_buf = get_output_buffer(input); - const auto& w_ih = get_output_buffer(w_ih_fwd); - size_t batch = in_buf.shape[0]; - size_t seq_len = in_buf.shape[1]; - size_t hidden_size = w_ih.shape[0] / 4; - return add_node(OpType::BILSTM_SEQUENCE, - {input, w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd, w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd}, - {batch, seq_len, 2 * hidden_size}, {}); -} - -size_t CactusGraph::maxpool1d(size_t input, size_t kernel_size, size_t stride) { - const auto& in_buf = get_output_buffer(input); - size_t batch = in_buf.shape[0]; - size_t channels = in_buf.shape[1]; - size_t input_length = in_buf.shape[2]; - size_t output_length = (input_length - kernel_size) / stride + 1; - - OpParams params; - params.kernel_size = kernel_size; - params.stride = stride; - return add_node(OpType::MAXPOOL1D, {input}, {batch, channels, output_length}, params); -} - -size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - if (w.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - const size_t C_out = w.shape[0]; - - if (w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s1p1 weight must match [C_out, C_in, 3, 3]"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S1P1, {input, weight}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::conv2d_k3s1p1(size_t input, size_t weight, size_t bias) { - const auto& xin = get_output_buffer(input); - const auto& w = get_output_buffer(weight); - const auto& b = get_output_buffer(bias); - - if (xin.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - - const size_t N = xin.shape[0]; - const size_t C_in = xin.shape[1]; - const size_t H = xin.shape[2]; - const size_t W = xin.shape[3]; - - if (w.shape.size() != 4 || w.shape[1] != C_in || w.shape[2] != 3 || w.shape[3] != 3) { - throw std::runtime_error("conv2d_k3s1p1 weight must be [C_out, C_in, 3, 3]"); - } - const size_t C_out = w.shape[0]; - if (b.total_size != C_out) { - throw std::runtime_error("conv2d_k3s1p1 bias size mismatch"); - } - if (H == 0 || W == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - OpParams params{}; - params.output_precision = xin.precision; - return add_node(OpType::CONV2D_K3S1P1, {input, weight, bias}, {N, C_out, H, W}, params); -} - -size_t CactusGraph::stats_pool(size_t input) { - const auto& xin = get_output_buffer(input); - size_t batch = xin.shape[0]; - size_t features = 1; - for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; - return add_node(OpType::STATS_POOL, {input}, {batch, features * 2}); -} - -size_t CactusGraph::weighted_stats_pool(size_t input, size_t weights) { - const auto& xin = get_output_buffer(input); - size_t batch = xin.shape[0]; - size_t features = 1; - for (size_t i = 1; i < xin.shape.size() - 1; ++i) features *= xin.shape[i]; - return add_node(OpType::WEIGHTED_STATS_POOL, {input, weights}, {batch, features * 2}); -} diff --git a/cactus/graph/graph_core.cpp b/cactus/graph/graph_core.cpp deleted file mode 100644 index f418da72b..000000000 --- a/cactus/graph/graph_core.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "graph.h" -#include -#include -#include - -size_t BufferPool::round_up_size(size_t size) const { - constexpr size_t ALIGNMENT = 64; - constexpr size_t MIN_BUCKET = 1024; - - if (size < MIN_BUCKET) return MIN_BUCKET; - - size_t aligned = (size + ALIGNMENT - 1) & ~(ALIGNMENT - 1); - - size_t bucket = MIN_BUCKET; - while (bucket < aligned) { - bucket *= 2; - } - return bucket; -} - -char* BufferPool::acquire(size_t byte_size) { - if (byte_size == 0) return nullptr; - - size_t bucket_size = round_up_size(byte_size); - - auto it = free_buffers_.find(bucket_size); - if (it != free_buffers_.end() && !it->second.empty()) { - auto buffer = std::move(it->second.back()); - it->second.pop_back(); - pool_bytes_ -= bucket_size; - active_bytes_ += bucket_size; - if (active_bytes_ > peak_bytes_) { - peak_bytes_ = active_bytes_; - } - return buffer.release(); - } - - active_bytes_ += bucket_size; - if (active_bytes_ > peak_bytes_) { - peak_bytes_ = active_bytes_; - } - return new char[bucket_size]; -} - -void BufferPool::release(char* ptr, size_t byte_size) { - if (!ptr || byte_size == 0) return; - - size_t bucket_size = round_up_size(byte_size); - active_bytes_ -= bucket_size; - pool_bytes_ += bucket_size; - - free_buffers_[bucket_size].push_back(std::unique_ptr(ptr)); -} - -void BufferPool::clear() { - free_buffers_.clear(); - pool_bytes_ = 0; -} - -BufferDesc::BufferDesc() - : total_size(0), byte_size(0), external_data(nullptr), pooled_data(nullptr), - precision(Precision::FP16) {} - -BufferDesc::BufferDesc(const std::vector& s, Precision prec) - : shape(s), external_data(nullptr), pooled_data(nullptr), precision(prec) { - total_size = 1; - for (size_t dim : shape) total_size *= dim; - byte_size = PrecisionTraits::packed_size_of(prec, total_size); -} - -BufferDesc::~BufferDesc() { - if (pooled_data) { - delete[] pooled_data; - pooled_data = nullptr; - } -} - -BufferDesc::BufferDesc(BufferDesc&& other) noexcept - : shape(std::move(other.shape)), - total_size(other.total_size), - byte_size(other.byte_size), - data(std::move(other.data)), - external_data(other.external_data), - pooled_data(other.pooled_data), - precision(other.precision), - group_size(other.group_size), - num_groups(other.num_groups), - scales_data(other.scales_data), - owned_scales(std::move(other.owned_scales)), - is_interleaved(other.is_interleaved), - original_N(other.original_N), - activation_scales_data(other.activation_scales_data), - owned_activation_scales(std::move(other.owned_activation_scales)), - num_rows_for_activation_scales(other.num_rows_for_activation_scales) { - other.total_size = 0; - other.byte_size = 0; - other.external_data = nullptr; - other.pooled_data = nullptr; - other.group_size = 0; - other.num_groups = 0; - other.scales_data = nullptr; - other.is_interleaved = false; - other.original_N = 0; - other.activation_scales_data = nullptr; - other.num_rows_for_activation_scales = 0; -} - -BufferDesc& BufferDesc::operator=(BufferDesc&& other) noexcept { - if (this != &other) { - if (pooled_data) { - delete[] pooled_data; - } - - shape = std::move(other.shape); - total_size = other.total_size; - byte_size = other.byte_size; - data = std::move(other.data); - external_data = other.external_data; - pooled_data = other.pooled_data; - precision = other.precision; - group_size = other.group_size; - num_groups = other.num_groups; - scales_data = other.scales_data; - owned_scales = std::move(other.owned_scales); - is_interleaved = other.is_interleaved; - original_N = other.original_N; - activation_scales_data = other.activation_scales_data; - owned_activation_scales = std::move(other.owned_activation_scales); - num_rows_for_activation_scales = other.num_rows_for_activation_scales; - - other.total_size = 0; - other.byte_size = 0; - other.external_data = nullptr; - other.pooled_data = nullptr; - other.group_size = 0; - other.num_groups = 0; - other.scales_data = nullptr; - other.is_interleaved = false; - other.original_N = 0; - other.activation_scales_data = nullptr; - other.num_rows_for_activation_scales = 0; - } - return *this; -} - -void* BufferDesc::get_data() { - if (external_data) return external_data; - if (pooled_data) return pooled_data; - return data.get(); -} - -const void* BufferDesc::get_data() const { - if (external_data) return external_data; - if (pooled_data) return pooled_data; - return data.get(); -} - -void BufferDesc::allocate() { - if (!data && !external_data && !pooled_data) { - data = std::make_unique(byte_size); - } -} - -void BufferDesc::allocate_from_pool(BufferPool& pool) { - if (!data && !external_data && !pooled_data && byte_size > 0) { - pooled_data = pool.acquire(byte_size); - } -} - -void BufferDesc::release_to_pool(BufferPool& pool) { - if (pooled_data && byte_size > 0) { - pool.release(pooled_data, byte_size); - pooled_data = nullptr; - } -} - -void BufferDesc::set_external(void* ptr) { - external_data = ptr; - data.reset(); - pooled_data = nullptr; -} - -// GraphNode implementation -GraphNode::GraphNode(size_t node_id, OpType type) : id(node_id), op_type(type) {} - -// TensorConfig implementation -TensorConfig& TensorConfig::global() { - static TensorConfig instance; - return instance; -} - -// BroadcastInfo implementation -BroadcastInfo BroadcastInfo::compute(const std::vector& lhs, const std::vector& rhs) { - BroadcastInfo info; - size_t max_dims = std::max(lhs.size(), rhs.size()); - info.output_shape.resize(max_dims); - - for (size_t i = 0; i < max_dims; ++i) { - size_t lhs_dim = i < lhs.size() ? lhs[lhs.size() - 1 - i] : 1; - size_t rhs_dim = i < rhs.size() ? rhs[rhs.size() - 1 - i] : 1; - - if (lhs_dim != rhs_dim && lhs_dim != 1 && rhs_dim != 1) { - throw std::invalid_argument("Shapes are not compatible for broadcasting"); - } - - info.output_shape[max_dims - 1 - i] = std::max(lhs_dim, rhs_dim); - } - - info.needs_broadcasting = (lhs != info.output_shape || rhs != info.output_shape); - return info; -} - -namespace ValidationUtils { - void validate_tensor_dims(const std::vector& shape, size_t required_dims, const std::string& op_name) { - if (shape.size() != required_dims) { - throw std::runtime_error(op_name + " requires " + std::to_string(required_dims) + - "D tensor, got " + std::to_string(shape.size()) + "D tensor"); - } - } - - void validate_precision(Precision actual, Precision required, const std::string& op_name) { - if (actual != required) { - std::string actual_str = (actual == Precision::INT8) ? "INT8" : "FP32"; - std::string required_str = (required == Precision::INT8) ? "INT8" : "FP32"; - throw std::runtime_error(op_name + " requires " + required_str + " precision, got " + actual_str); - } - } - - void validate_input_count(size_t actual, size_t required, const std::string& op_name) { - if (actual < required) { - throw std::runtime_error(op_name + " requires " + std::to_string(required) + - " inputs, got " + std::to_string(actual) + " inputs"); - } - } -} - -CactusGraph::CactusGraph() : next_node_id_(1) {} - -size_t CactusGraph::get_node_count() const { - return nodes_.size(); -} - -void CactusGraph::register_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { - debug_nodes_.push_back({layer_idx, name, node_id}); -} - -void CactusGraph::capture_debug_node(uint32_t layer_idx, const std::string& name, size_t node_id) { - register_debug_node(layer_idx, name, node_id); -} - -const std::vector& CactusGraph::get_debug_nodes() const { - return debug_nodes_; -} - -void CactusGraph::clear_debug_nodes() { - debug_nodes_.clear(); -} - -void CactusGraph::allocate_buffers() { - for (auto& node : nodes_) { - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate(); - } - } -} diff --git a/cactus/graph/graph_execute.cpp b/cactus/graph/graph_execute.cpp deleted file mode 100644 index f8bbc11a5..000000000 --- a/cactus/graph/graph_execute.cpp +++ /dev/null @@ -1,806 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using ComputeFn = void(*)(GraphNode&, const nodes_vector&, const node_index_map_t&); - -#define DECLARE_COMPUTE(name) \ - extern void name(GraphNode&, const nodes_vector&, const node_index_map_t&) - -DECLARE_COMPUTE(compute_binary_op_node); -DECLARE_COMPUTE(compute_unary_op_node); -DECLARE_COMPUTE(compute_activation_node); -DECLARE_COMPUTE(compute_reduce_node); -DECLARE_COMPUTE(compute_reshape_node); -DECLARE_COMPUTE(compute_precision_cast_node); -DECLARE_COMPUTE(compute_matmul_node); -DECLARE_COMPUTE(compute_rms_norm_node); -DECLARE_COMPUTE(compute_rope_node); -DECLARE_COMPUTE(compute_softmax_node); -DECLARE_COMPUTE(compute_attention_node); -DECLARE_COMPUTE(compute_attention_int8_hybrid_node); -DECLARE_COMPUTE(compute_rel_pos_bias_node); -DECLARE_COMPUTE(compute_layernorm_node); -DECLARE_COMPUTE(compute_conv1d_causal_node); -DECLARE_COMPUTE(compute_conv1d_k3_node); -DECLARE_COMPUTE(compute_conv1d_k7s3_node); -DECLARE_COMPUTE(compute_conv1d_node); -DECLARE_COMPUTE(compute_conv1d_same_depthwise_k9_node); -DECLARE_COMPUTE(compute_conv1d_pointwise_node); -DECLARE_COMPUTE(compute_conv2d_k3s2p1_node); -DECLARE_COMPUTE(compute_conv2d_depthwise_k3s2p1_node); -DECLARE_COMPUTE(compute_conv2d_pointwise_1x1_node); -DECLARE_COMPUTE(compute_glu_node); -DECLARE_COMPUTE(compute_batchnorm_node); -DECLARE_COMPUTE(compute_groupnorm_node); -DECLARE_COMPUTE(compute_rope_gptj_node); -DECLARE_COMPUTE(compute_lstm_cell_node); -DECLARE_COMPUTE(compute_gated_deltanet_decode_node); -DECLARE_COMPUTE(compute_gated_deltanet_prefill_node); -DECLARE_COMPUTE(compute_stft_node); -DECLARE_COMPUTE(compute_altup_predict_node); -DECLARE_COMPUTE(compute_altup_correct_node); -DECLARE_COMPUTE(compute_gaussian_topk_node); -DECLARE_COMPUTE(compute_maxpool1d_node); -DECLARE_COMPUTE(compute_bilstm_sequence_node); -DECLARE_COMPUTE(compute_conv2d_k3s1p1_node); -DECLARE_COMPUTE(compute_stats_pool_node); -DECLARE_COMPUTE(compute_transpose_node); -DECLARE_COMPUTE(compute_gather_node); -DECLARE_COMPUTE(compute_slice_node); -DECLARE_COMPUTE(compute_embedding_node); -DECLARE_COMPUTE(compute_concat_node); -DECLARE_COMPUTE(compute_cat_node); -DECLARE_COMPUTE(compute_index_node); -DECLARE_COMPUTE(compute_bilinear_interpolation_node); -DECLARE_COMPUTE(compute_sample_node); -DECLARE_COMPUTE(compute_topk_node); -DECLARE_COMPUTE(compute_scatter_topk_node); -DECLARE_COMPUTE(compute_moe_layer_node); -DECLARE_COMPUTE(compute_dense_mlp_int4_fused_node); -DECLARE_COMPUTE(compute_persistent_node); -DECLARE_COMPUTE(compute_quantize_activations_node); -extern void shrink_thread_local_buffers(); -#undef DECLARE_COMPUTE - -static const std::unordered_map dispatch_table = { - {OpType::ADD, compute_binary_op_node}, - {OpType::ADD_CLIPPED, compute_binary_op_node}, - {OpType::SUBTRACT, compute_binary_op_node}, - {OpType::MULTIPLY, compute_binary_op_node}, - {OpType::DIVIDE, compute_binary_op_node}, - {OpType::SCALAR_ADD, compute_unary_op_node}, - {OpType::SCALAR_SUBTRACT, compute_unary_op_node}, - {OpType::SCALAR_MULTIPLY, compute_unary_op_node}, - {OpType::SCALAR_DIVIDE, compute_unary_op_node}, - {OpType::SCALAR_EXP, compute_unary_op_node}, - {OpType::SCALAR_SQRT, compute_unary_op_node}, - {OpType::SCALAR_COS, compute_unary_op_node}, - {OpType::SCALAR_SIN, compute_unary_op_node}, - {OpType::SCALAR_LOG, compute_unary_op_node}, - {OpType::ABS, compute_unary_op_node}, - {OpType::POW, compute_unary_op_node}, - {OpType::RELU, compute_activation_node}, - {OpType::SILU, compute_activation_node}, - {OpType::GELU, compute_activation_node}, - {OpType::GELU_ERF, compute_activation_node}, - {OpType::SIGMOID, compute_activation_node}, - {OpType::TANH, compute_activation_node}, - {OpType::LEAKY_RELU, compute_activation_node}, - {OpType::SUM, compute_reduce_node}, - {OpType::MEAN, compute_reduce_node}, - {OpType::VARIANCE, compute_reduce_node}, - {OpType::MIN, compute_reduce_node}, - {OpType::MAX, compute_reduce_node}, - {OpType::FLATTEN, compute_reshape_node}, - {OpType::VIEW, compute_reshape_node}, - {OpType::RESHAPE, compute_reshape_node}, - {OpType::PRECISION_CAST, compute_precision_cast_node}, - {OpType::MATMUL, compute_matmul_node}, - {OpType::RMS_NORM, compute_rms_norm_node}, - {OpType::LAYERNORM, compute_layernorm_node}, - {OpType::GROUPNORM, compute_groupnorm_node}, - {OpType::BATCHNORM, compute_batchnorm_node}, - {OpType::ROPE, compute_rope_node}, - {OpType::ROPE_GPTJ, compute_rope_gptj_node}, - {OpType::SOFTMAX, compute_softmax_node}, - {OpType::ATTENTION, compute_attention_node}, - {OpType::ATTENTION_INT8_HYBRID, compute_attention_int8_hybrid_node}, - {OpType::REL_POS_BIAS, compute_rel_pos_bias_node}, - {OpType::CONV1D_CAUSAL, compute_conv1d_causal_node}, - {OpType::CONV1D_K3, compute_conv1d_k3_node}, - {OpType::CONV1D_K7S3, compute_conv1d_k7s3_node}, - {OpType::CONV1D, compute_conv1d_node}, - {OpType::CONV1D_SAME_DEPTHWISE_K9, compute_conv1d_same_depthwise_k9_node}, - {OpType::CONV1D_POINTWISE, compute_conv1d_pointwise_node}, - {OpType::CONV2D_K3S2P1, compute_conv2d_k3s2p1_node}, - {OpType::CONV2D_DEPTHWISE_K3S2P1, compute_conv2d_depthwise_k3s2p1_node}, - {OpType::CONV2D_POINTWISE_1X1, compute_conv2d_pointwise_1x1_node}, - {OpType::CONV2D_K3S1P1, compute_conv2d_k3s1p1_node}, - {OpType::GLU, compute_glu_node}, - {OpType::TRANSPOSE, compute_transpose_node}, - {OpType::GATHER, compute_gather_node}, - {OpType::SLICE, compute_slice_node}, - {OpType::EMBEDDING, compute_embedding_node}, - {OpType::CONCAT, compute_concat_node}, - {OpType::CAT, compute_cat_node}, - {OpType::INDEX, compute_index_node}, - {OpType::BILINEAR_INTERPOLATION, compute_bilinear_interpolation_node}, - {OpType::SAMPLE, compute_sample_node}, - {OpType::TOPK, compute_topk_node}, - {OpType::SCATTER_TOPK, compute_scatter_topk_node}, - {OpType::MOE_LAYER, compute_moe_layer_node}, - {OpType::DENSE_MLP_INT4_FUSED, compute_dense_mlp_int4_fused_node}, - {OpType::PERSISTENT, compute_persistent_node}, - {OpType::QUANTIZE_ACTIVATIONS, compute_quantize_activations_node}, - {OpType::LSTM_CELL, compute_lstm_cell_node}, - {OpType::GATED_DELTANET_DECODE, compute_gated_deltanet_decode_node}, - {OpType::GATED_DELTANET_PREFILL, compute_gated_deltanet_prefill_node}, - {OpType::STFT, compute_stft_node}, - {OpType::ALTUP_PREDICT, compute_altup_predict_node}, - {OpType::ALTUP_CORRECT, compute_altup_correct_node}, - {OpType::GAUSSIAN_TOPK, compute_gaussian_topk_node}, - {OpType::MAXPOOL1D, compute_maxpool1d_node}, - {OpType::BILSTM_SEQUENCE, compute_bilstm_sequence_node}, - {OpType::STATS_POOL, compute_stats_pool_node}, - {OpType::WEIGHTED_STATS_POOL, compute_weighted_stats_pool_node}, -}; - -static const char* op_type_names[] = { - "INPUT", "PRECISION_CAST", - "ADD", "ADD_CLIPPED", "SUBTRACT", "MULTIPLY", "DIVIDE", - "MATMUL", "TRANSPOSE", "RESHAPE", "SLICE", "GATHER", "EMBEDDING", "VIEW", "FLATTEN", - "BILINEAR_INTERPOLATION", - "SUM", "MEAN", "VARIANCE", "MIN", "MAX", - "RMS_NORM", "ROPE", "ROPE_GPTJ", "SOFTMAX", "ATTENTION", "ATTENTION_INT8_HYBRID", "REL_POS_BIAS", "CONV1D_CAUSAL", "CONV1D_K3", "CONV1D_K7S3", "CONV1D", "CONV1D_SAME_DEPTHWISE_K9", "CONV1D_POINTWISE", "CONV2D_K3S2P1", "CONV2D_DEPTHWISE_K3S2P1", "CONV2D_POINTWISE_1X1", "GLU", "BATCHNORM", - "SCALAR_ADD", "SCALAR_SUBTRACT", "SCALAR_MULTIPLY", "SCALAR_DIVIDE", - "SCALAR_EXP", "SCALAR_SQRT", "SCALAR_COS", "SCALAR_SIN", "SCALAR_LOG", - "ABS", "POW", - "RELU", "SILU", "GELU", "GELU_ERF", "SIGMOID", "TANH", - "SAMPLE", "CONCAT", - "SCATTER_TOPK", - "TOPK", "LAYERNORM", "GROUPNORM", - "MOE_LAYER", - "INDEX", - "PERSISTENT", - "QUANTIZE_ACTIVATIONS", - "LSTM_CELL", - "GATED_DELTANET_DECODE", - "GATED_DELTANET_PREFILL", - "STFT", - "ALTUP_PREDICT", - "ALTUP_CORRECT", - "GAUSSIAN_TOPK", - "MAXPOOL1D", - "BILSTM_SEQUENCE", - "LEAKY_RELU", - "CONV2D_K3S1P1", - "STATS_POOL", - "WEIGHTED_STATS_POOL", - "DENSE_MLP_INT4_FUSED" -}; - -static const char* get_op_name(OpType op) { - return op_type_names[static_cast(op)]; -} - -void compute_node_optimized(GraphNode& node, const nodes_vector& nodes, const node_index_map_t& node_index_map) { - if (node.op_type == OpType::INPUT) return; - - auto it = dispatch_table.find(node.op_type); - if (it != dispatch_table.end()) { - it->second(node, nodes, node_index_map); - } else { - throw std::runtime_error("Unknown operation type: " + std::to_string(static_cast(node.op_type))); - } -} - -void CactusGraph::set_input(size_t node_id, const void* data, Precision) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); - } - - auto& node = *nodes_[it->second]; - if (node.op_type != OpType::INPUT) { - throw std::invalid_argument("Can only set data on input nodes"); - } - - if (!node.output_buffer.data && !node.output_buffer.external_data) { - node.output_buffer.allocate(); - } - - std::memcpy(node.output_buffer.get_data(), data, node.output_buffer.byte_size); -} - -void CactusGraph::set_external_input(size_t node_id, void* data, Precision) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown input node id: " + std::to_string(node_id)); - } - - auto& node = *nodes_[it->second]; - if (node.op_type != OpType::INPUT) { - throw std::invalid_argument("Can only set data on input nodes"); - } - - node.output_buffer.set_external(data); -} - -void* CactusGraph::get_output(size_t node_id) { - auto it = node_index_map_.find(node_id); - if (it == node_index_map_.end()) { - throw std::out_of_range("Unknown output node id: " + std::to_string(node_id)); - } - - auto& buffer = nodes_[it->second]->output_buffer; - if (!buffer.get_data()) { - buffer.allocate(); - } - return buffer.get_data(); -} - -void CactusGraph::execute(const std::string& profile_file) { - std::vector last_use(nodes_.size(), 0); - for (size_t i = 0; i < nodes_.size(); ++i) { - for (size_t input_id : nodes_[i]->input_ids) { - auto it = node_index_map_.find(input_id); - if (it != node_index_map_.end()) { - last_use[it->second] = std::max(last_use[it->second], i); - } - } - } - - BufferPool& pool = buffer_pool_; - - auto get_env_int = [](const char* name, int fallback) -> int { - const char* val = std::getenv(name); - return val ? std::atoi(val) : fallback; - }; - - auto get_env_str = [](const char* name) -> std::string { - const char* val = std::getenv(name); - return val ? std::string(val) : std::string(); - }; - - bool capture_to_stdout = get_env_int("CACTUS_CAPTURE_STDOUT", 0) != 0; - std::string capture_file_path = get_env_str("CACTUS_CAPTURE_FILE"); - bool capture_requested = get_env_int("CACTUS_CAPTURE_ENABLE", 0) != 0; - std::string capture_dir = get_env_str("CACTUS_CAPTURE_DIR"); - - if (!capture_requested) { - capture_requested = capture_to_stdout || !capture_file_path.empty() || !capture_dir.empty(); - } else if (capture_file_path.empty() && !capture_to_stdout && capture_dir.empty()) { - capture_to_stdout = true; - } - - size_t capture_preview_count = static_cast(get_env_int("CACTUS_CAPTURE_PREVIEW_COUNT", 8)); - size_t capture_max_elements = static_cast(get_env_int("CACTUS_CAPTURE_MAX_ELEMENTS", 65536)); - - std::string env_profile = get_env_str("CACTUS_PROFILE_FILE"); - if (env_profile.empty()) env_profile = get_env_str("CACTUS_PROFILE"); - - std::string target_profile = profile_file; - if (target_profile.empty() && !env_profile.empty()) { - target_profile = env_profile; - } - - bool enable_profiling = !target_profile.empty(); - bool to_stdout = (target_profile == "stdout" || target_profile == "-"); - - std::ofstream profile_out; - std::ostream* out = &std::cout; - - if (enable_profiling && !to_stdout) { - profile_out.open(target_profile, std::ios::app); - if (profile_out.is_open()) { - out = &profile_out; - } - } - - auto total_start = std::chrono::high_resolution_clock::now(); - - if (enable_profiling) { - *out << "=== Graph Execution Profile ===" << std::endl; - *out << std::left << std::setw(24) << "Operation" - << std::setw(12) << "Time (ms)" - << std::setw(20) << "Output Shape" - << "Backend" << std::endl; - *out << std::string(72, '-') << std::endl; - } - - for (size_t node_idx = 0; node_idx < nodes_.size(); ++node_idx) { - auto& node = nodes_[node_idx]; - - if (node->op_type != OpType::INPUT) { - node->output_buffer.allocate_from_pool(pool); - } - - if (enable_profiling && node->op_type != OpType::INPUT) { - auto start = std::chrono::high_resolution_clock::now(); - - compute_node_optimized(*node, nodes_, node_index_map_); - - if (node->op_type == OpType::PERSISTENT) { - populated_node_ids_.insert(node->id); - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - double ms = duration.count() / 1000.0; - - std::string shape_str = "["; - for (size_t i = 0; i < node->output_buffer.shape.size(); ++i) { - if (i > 0) shape_str += ","; - shape_str += std::to_string(node->output_buffer.shape[i]); - } - shape_str += "]"; - - std::string values_str = ""; - if (node->output_buffer.get_data()) { - size_t num_values = std::min(size_t(5), node->output_buffer.total_size); - values_str = " values=["; - - if (node->output_buffer.precision == Precision::FP32) { - if (node->op_type == OpType::SAMPLE) { - uint32_t* uint32_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(uint32_data[i]); - } - } else { - float* float_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(float_data[i]).substr(0, 6); - } - } - } else if (node->output_buffer.precision == Precision::FP16) { - __fp16* fp16_data = reinterpret_cast<__fp16*>(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); - } - } else if (node->output_buffer.precision == Precision::INT8) { - int8_t* int8_data = reinterpret_cast(node->output_buffer.get_data()); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) values_str += ","; - values_str += std::to_string(static_cast(int8_data[i])); - } - } - - if (node->output_buffer.total_size > 5) { - values_str += ",..."; - } - values_str += "]"; - } - - std::string weights_str = ""; - if ((node->op_type == OpType::RMS_NORM || node->op_type == OpType::MATMUL || - node->op_type == OpType::GATHER || node->op_type == OpType::EMBEDDING || - node->op_type == OpType::ATTENTION || node->op_type == OpType::CONCAT) && - node->input_ids.size() >= 2) { - const auto& weight_node = nodes_[node_index_map_.at(node->input_ids[1])]; - if (weight_node->output_buffer.get_data()) { - size_t num_values = std::min(size_t(5), weight_node->output_buffer.total_size); - weights_str = " weights=["; - - if (weight_node->output_buffer.precision == Precision::FP32) { - const float* float_data = weight_node->output_buffer.data_as(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(float_data[i]).substr(0, 6); - } - } else if (weight_node->output_buffer.precision == Precision::FP16) { - const __fp16* fp16_data = weight_node->output_buffer.data_as<__fp16>(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(static_cast(fp16_data[i])).substr(0, 6); - } - } else if (weight_node->output_buffer.precision == Precision::INT8) { - const int8_t* int8_data = weight_node->output_buffer.data_as(); - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - weights_str += std::to_string(static_cast(int8_data[i])); - } - } else if (weight_node->output_buffer.precision == Precision::INT4) { - const uint8_t* packed = weight_node->output_buffer.data_as(); - int8x16_t high, low; - unpack_int4_as_int8x16x2(packed, high, low); - int8_t low_lanes[16], high_lanes[16]; - vst1q_s8(low_lanes, low); - vst1q_s8(high_lanes, high); - - for (size_t i = 0; i < num_values; ++i) { - if (i > 0) weights_str += ","; - int8_t val = (i < 16) ? low_lanes[i] : high_lanes[i - 16]; - weights_str += std::to_string(static_cast(val)); - } - } - - if (weight_node->output_buffer.total_size > 5) { - weights_str += ",..."; - } - weights_str += "]"; - } - } - - *out << std::left << std::setw(24) << get_op_name(node->op_type) - << std::setw(12) << std::fixed << std::setprecision(3) << ms - << std::setw(20) << shape_str - << values_str << weights_str << std::endl; - } else { - compute_node_optimized(*node, nodes_, node_index_map_); - - if (node->op_type == OpType::PERSISTENT) { - populated_node_ids_.insert(node->id); - } - } - } - - std::unique_ptr capture_file_stream; - std::vector capture_outputs; - - if (capture_requested) { - if (capture_to_stdout) { - capture_outputs.push_back(&std::cout); - } - - if (!capture_file_path.empty()) { - std::filesystem::path capture_path(capture_file_path); - if (capture_path.has_parent_path()) { - std::error_code ec; - std::filesystem::create_directories(capture_path.parent_path(), ec); - } - - auto stream_ptr = std::make_unique(capture_path, std::ios::out | std::ios::app); - if (stream_ptr->is_open()) { - capture_outputs.push_back(stream_ptr.get()); - capture_file_stream = std::move(stream_ptr); - } else { - std::cerr << "Failed to open capture file: " << capture_path << std::endl; - } - } - - if (!capture_dir.empty()) { - std::filesystem::path dir_path(capture_dir); - std::error_code ec; - std::filesystem::create_directories(dir_path, ec); - } - - if (capture_outputs.empty() && capture_dir.empty()) { - capture_requested = false; - } - } - - if (capture_requested) { - auto precision_to_string = [](Precision p) -> const char* { - switch (p) { - case Precision::FP32: return "FP32"; - case Precision::FP16: return "FP16"; - case Precision::INT8: return "INT8"; - default: return "UNKNOWN"; - } - }; - - auto format_double = [](double value) { - std::ostringstream oss; - oss << std::fixed << std::setprecision(6) << value; - return oss.str(); - }; - - auto now = std::chrono::system_clock::now(); - std::time_t now_time = std::chrono::system_clock::to_time_t(now); - std::tm time_info{}; -#if defined(_WIN32) - localtime_s(&time_info, &now_time); -#else - localtime_r(&now_time, &time_info); -#endif - - auto write_header = [&](std::ostream& stream) { - stream << "=== Graph Debug Capture ===" << std::endl; - stream << "Timestamp: " << std::put_time(&time_info, "%Y-%m-%d %H:%M:%S") << std::endl; - stream << "Captured nodes: " << debug_nodes_.size() << std::endl; - stream << std::string(60, '-') << std::endl; - }; - - auto write_separator = [](std::ostream& stream) { - stream << std::string(60, '-') << std::endl; - }; - - if (debug_nodes_.empty()) { - for (auto* stream : capture_outputs) { - write_header(*stream); - *stream << "No debug nodes registered on this graph." << std::endl; - write_separator(*stream); - stream->flush(); - } - } else { - for (auto* stream : capture_outputs) { - write_header(*stream); - } - - for (const auto& entry : debug_nodes_) { - auto node_it = node_index_map_.find(entry.node_id); - const GraphNode* node_ptr = nullptr; - if (node_it != node_index_map_.end()) { - node_ptr = nodes_[node_it->second].get(); - } - - if (!node_ptr) { - for (auto* stream : capture_outputs) { - *stream << "Layer " << entry.layer_idx << " - " << entry.name - << " (node " << entry.node_id << ")" << std::endl; - *stream << " Data: " << std::endl; - write_separator(*stream); - } - continue; - } - - const BufferDesc& buffer = node_ptr->output_buffer; - const void* data_ptr = buffer.get_data(); - size_t total_size = buffer.total_size; - - std::ostringstream shape_ss; - shape_ss << "["; - for (size_t i = 0; i < buffer.shape.size(); ++i) { - if (i > 0) { - shape_ss << ","; - } - shape_ss << buffer.shape[i]; - } - shape_ss << "]"; - std::string shape_str = shape_ss.str(); - - bool has_data = data_ptr != nullptr && total_size > 0; - size_t elements_to_process = total_size; - bool truncated = false; - if (has_data && elements_to_process > capture_max_elements && capture_max_elements > 0) { - elements_to_process = capture_max_elements; - truncated = true; - } - - std::vector preview_values; - if (capture_preview_count > 0) { - preview_values.reserve(std::min(capture_preview_count, elements_to_process)); - } - - double min_val = std::numeric_limits::infinity(); - double max_val = -std::numeric_limits::infinity(); - long double sum = 0.0L; - long double sum_sq = 0.0L; - - if (has_data && elements_to_process > 0) { - auto accumulate = [&](float value, size_t index) { - double v = static_cast(value); - min_val = std::min(min_val, v); - max_val = std::max(max_val, v); - sum += static_cast(value); - sum_sq += static_cast(value) * static_cast(value); - if (capture_preview_count > 0 && index < capture_preview_count) { - preview_values.push_back(value); - } - }; - - if (buffer.precision == Precision::FP32) { - const float* typed = static_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(typed[i], i); - } - } else if (buffer.precision == Precision::FP16) { - const __fp16* typed = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(static_cast(typed[i]), i); - } - } else if (buffer.precision == Precision::INT8) { - const int8_t* typed = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; ++i) { - accumulate(static_cast(typed[i]), i); - } - } else if (buffer.precision == Precision::INT4) { - assert(elements_to_process % 32 == 0 && "INT4 precision capture requires element count to be multiple of 32"); - const uint8_t* packed_ptr = reinterpret_cast(data_ptr); - for (size_t i = 0; i < elements_to_process; i+=32) { - int8x16_t high, low; - unpack_int4_as_int8x16x2(packed_ptr + i / 2, high, low); - int8_t high_lanes[16], low_lanes[16]; - vst1q_s8(high_lanes, high); - vst1q_s8(low_lanes, low); - - for (size_t j = 0; j < 16; ++j) { - accumulate(static_cast(low_lanes[j]), i + j); - } - for (size_t j = 0; j < 16; ++j) { - accumulate(static_cast(high_lanes[j]), i + 16 + j); - } - } - } else { - has_data = false; - } - } else { - has_data = false; - } - - if (!capture_dir.empty() && has_data) { - std::string safe_name = entry.name; - std::string filename = capture_dir + "/" + safe_name + ".bin"; - std::ofstream bin_file(filename, std::ios::binary); - if (bin_file.is_open()) { - size_t bytes_to_write = buffer.byte_size; - if (truncated) { - bytes_to_write = PrecisionTraits::packed_size_of(buffer.precision, elements_to_process); - } - bin_file.write(reinterpret_cast(data_ptr), bytes_to_write); - } - } - - size_t processed_count = has_data ? elements_to_process : 0; - long double mean_ld = processed_count > 0 ? sum / processed_count : 0.0L; - long double variance_ld = processed_count > 0 ? (sum_sq / processed_count) - (mean_ld * mean_ld) : 0.0L; - if (variance_ld < 0.0L) { - variance_ld = 0.0L; - } - double mean_val = static_cast(mean_ld); - double stddev_val = processed_count > 0 ? std::sqrt(static_cast(variance_ld)) : 0.0; - - std::ostringstream preview_ss; - if (capture_preview_count > 0 && !preview_values.empty()) { - preview_ss << "["; - for (size_t i = 0; i < preview_values.size(); ++i) { - if (i > 0) { - preview_ss << ", "; - } - preview_ss << format_double(static_cast(preview_values[i])); - } - if (processed_count > preview_values.size()) { - if (!preview_values.empty()) { - preview_ss << ", ..."; - } else { - preview_ss << "..."; - } - } - preview_ss << "]"; - } - - for (auto* stream : capture_outputs) { - *stream << "Layer " << entry.layer_idx << " - " << entry.name - << " (node " << entry.node_id << ")" << std::endl; - *stream << " Shape: " << shape_str << " Precision: " << precision_to_string(buffer.precision) << std::endl; - if (!has_data) { - *stream << " Data: " << std::endl; - } else { - *stream << " Stats: min=" << format_double(min_val) - << " max=" << format_double(max_val) - << " mean=" << format_double(mean_val) - << " std=" << format_double(stddev_val) << std::endl; - if (truncated || processed_count < total_size) { - *stream << " Note: stats computed on first " << processed_count - << " of " << total_size << " values" << std::endl; - } - if (capture_preview_count > 0 && !preview_values.empty()) { - *stream << " Preview: " << preview_ss.str() << std::endl; - } - } - write_separator(*stream); - } - } - - for (auto* stream : capture_outputs) { - stream->flush(); - } - } - } - - if (enable_profiling) { - auto total_end = std::chrono::high_resolution_clock::now(); - auto total_duration = std::chrono::duration_cast(total_end - total_start); - double total_ms = total_duration.count() / 1000.0; - - *out << std::string(72, '-') << std::endl; - *out << "Total execution time: " << std::fixed << std::setprecision(3) << total_ms << " ms" << std::endl; - *out << "================================" << std::endl; - - if (profile_out.is_open()) { - profile_out.close(); - } - } -} - -void CactusGraph::hard_reset() { - nodes_.clear(); - node_index_map_.clear(); - mapped_files_.clear(); - weight_cache_.clear(); - next_node_id_ = 1; - debug_nodes_.clear(); - buffer_pool_.clear(); -} - -void CactusGraph::soft_reset() { - std::set cached_node_ids; - for (const auto& cache_entry : weight_cache_) { - cached_node_ids.insert(cache_entry.second); - } - - for (size_t pid : persistent_node_ids_) { - cached_node_ids.insert(pid); - } - - size_t max_preserved_id = 0; - for (const auto& node : nodes_) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - max_preserved_id = std::max(max_preserved_id, node->id); - } - } - - auto preserved_nodes = std::move(nodes_); - auto preserved_index_map = std::move(node_index_map_); - - nodes_.clear(); - node_index_map_.clear(); - - for (auto& node : preserved_nodes) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - size_t index = nodes_.size(); - node_index_map_[node->id] = index; - nodes_.push_back(std::move(node)); - } - } - - next_node_id_ = max_preserved_id + 1; - debug_nodes_.clear(); - if (!prefill_mode_) { - buffer_pool_.clear(); - shrink_thread_local_buffers(); - } -} - -void CactusGraph::soft_reset_keep_pool() { - std::set cached_node_ids; - for (const auto& cache_entry : weight_cache_) { - cached_node_ids.insert(cache_entry.second); - } - - for (size_t pid : persistent_node_ids_) { - cached_node_ids.insert(pid); - } - - size_t max_preserved_id = 0; - for (const auto& node : nodes_) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - max_preserved_id = std::max(max_preserved_id, node->id); - } - } - - auto preserved_nodes = std::move(nodes_); - - nodes_.clear(); - node_index_map_.clear(); - - for (auto& node : preserved_nodes) { - if ((node->op_type == OpType::INPUT && node->output_buffer.external_data) || - cached_node_ids.count(node->id)) { - size_t index = nodes_.size(); - node_index_map_[node->id] = index; - nodes_.push_back(std::move(node)); - } - } - - next_node_id_ = max_preserved_id + 1; - debug_nodes_.clear(); -} diff --git a/cactus/graph/graph_io.cpp b/cactus/graph/graph_io.cpp deleted file mode 100644 index 8eded5e94..000000000 --- a/cactus/graph/graph_io.cpp +++ /dev/null @@ -1,808 +0,0 @@ -#include "graph.h" -#include "graph_param_io.h" -#include -#include -#include -#include -#include -#include -#include - -namespace { - constexpr uint32_t fourcc(char a, char b, char c, char d) { - return static_cast(static_cast(a)) | - (static_cast(static_cast(b)) << 8) | - (static_cast(static_cast(c)) << 16) | - (static_cast(static_cast(d)) << 24); - } - - constexpr uint32_t CACTUS_MAGIC = 0x54434143; - constexpr uint32_t CACTUS_GRAPH_MAGIC = fourcc('C', 'G', 'R', 'F'); - constexpr uint32_t FLAG_HAS_SCALES = 1 << 0; - constexpr uint32_t FLAG_INTERLEAVED = 1 << 3; - constexpr size_t HEADER_SIZE = 84; - - inline size_t align_offset(size_t offset, size_t alignment) { - size_t remainder = offset % alignment; - if (remainder == 0) return offset; - return offset + (alignment - remainder); - } - - inline void write_u32(std::ostream& out, uint32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - - inline void write_u64(std::ostream& out, uint64_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); - } - - void write_size_vector(std::ostream& out, const std::vector& values) { - uint32_t size = static_cast(values.size()); - write_u32(out, size); - for (size_t v : values) { - write_u64(out, static_cast(v)); - } - } - - void write_u32_vector(std::ostream& out, const std::vector& values) { - uint32_t size = static_cast(values.size()); - write_u32(out, size); - for (uint32_t v : values) { - write_u32(out, v); - } - } - - std::vector compute_leaf_outputs(const GraphFile::SerializedGraph& sg) { - std::vector referenced(sg.nodes.size(), false); - - for (const auto& node : sg.nodes) { - for (uint32_t input_idx : node.inputs) { - if (input_idx >= referenced.size()) { - throw std::runtime_error("Graph save failed: input index out of range"); - } - referenced[input_idx] = true; - } - } - - std::vector outputs; - outputs.reserve(sg.nodes.size()); - - for (uint32_t i = 0; i < sg.nodes.size(); ++i) { - if (!referenced[i]) { - outputs.push_back(i); - } - } - - return outputs; - } - - void write_serialized_graph(std::ostream& out, const GraphFile::SerializedGraph& sg) { - write_u32(out, sg.header.magic); - write_u32(out, sg.header.version); - write_u32(out, sg.header.node_count); - write_u32(out, sg.header.flags); - - write_u32_vector(out, sg.graph_inputs); - write_u32_vector(out, sg.graph_outputs); - - for (const auto& node : sg.nodes) { - write_u32(out, node.index); - write_u32(out, static_cast(node.op_type)); - write_u32_vector(out, node.inputs); - write_size_vector(out, node.output_shape); - write_u32(out, static_cast(node.precision)); - GraphParamIO::write_op_params(out, node.op_type, node.params); - } - - if (!out) { - throw std::runtime_error("Error writing serialized graph"); - } - } - - // read helpers - uint32_t read_u32(std::istream& in) { - uint32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading uint32"); - } - return v; - } - - uint64_t read_u64(std::istream& in) { - uint64_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) { - throw std::runtime_error("Unexpected EOF while reading uint64"); - } - return v; - } - - std::vector read_u32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) { - values.push_back(read_u32(in)); - } - return values; - } - - std::vector read_size_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) { - values.push_back(static_cast(read_u64(in))); - } - return values; - } - - bool is_binary_broadcast_op(OpType op_type) { - switch (op_type) { - case OpType::ADD: - case OpType::ADD_CLIPPED: - case OpType::SUBTRACT: - case OpType::MULTIPLY: - case OpType::DIVIDE: - return true; - default: - return false; - } - } - - void populate_derived_params(CactusGraph& graph, - const GraphFile::NodeEntry& node_entry, - const std::vector& runtime_inputs, - OpParams& params) { - if (is_binary_broadcast_op(node_entry.op_type)) { - if (runtime_inputs.size() != 2) { - throw std::runtime_error("Graph file corrupted: binary op missing inputs"); - } - const auto& lhs = graph.get_output_buffer(runtime_inputs[0]); - const auto& rhs = graph.get_output_buffer(runtime_inputs[1]); - params.broadcast_info = BroadcastInfo::compute(lhs.shape, rhs.shape); - } - } - - GraphFile::GraphHeader read_graph_header(std::istream& in) { - GraphFile::GraphHeader header; - header.magic = read_u32(in); - header.version = read_u32(in); - header.node_count = read_u32(in); - header.flags = read_u32(in); - - if (header.magic != CACTUS_GRAPH_MAGIC) { - throw std::runtime_error("Invalid graph file: bad magic"); - } - if (header.version != 4) { - throw std::runtime_error("Unsupported graph file version: " + - std::to_string(header.version)); - } - - return header; - } - - GraphFile::NodeEntry read_node_entry(std::istream& in) { - GraphFile::NodeEntry node; - node.index = read_u32(in); - uint32_t op_type_val = read_u32(in); - if (op_type_val > static_cast(OpType::STATS_POOL)) { - throw std::runtime_error("Graph file corrupted: invalid op type"); - } - node.op_type = static_cast(op_type_val); - node.inputs = read_u32_vector(in); - node.output_shape = read_size_vector(in); - uint32_t precision_val = read_u32(in); - if (precision_val > static_cast(Precision::INT4)) { - throw std::runtime_error("Graph file corrupted: invalid precision"); - } - node.precision = static_cast(precision_val); - GraphParamIO::read_op_params(in, node.op_type, node.params); - return node; - } - - -} // namespace - -void CactusGraph::save(const std::string& path) { - GraphFile::save_graph(*this, path); -} - -CactusGraph CactusGraph::from_serialized(const GraphFile::SerializedGraph& sg) { - CactusGraph graph; - std::vector runtime_ids; - runtime_ids.reserve(sg.nodes.size()); - - if (sg.nodes.size() != sg.header.node_count) { - throw std::runtime_error("Graph file corrupted: node count mismatch"); - } - - for (size_t i = 0; i < sg.nodes.size(); ++i) { - const auto& node_entry = sg.nodes[i]; - - if (node_entry.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); - } - std::vector runtime_inputs; - runtime_inputs.reserve(node_entry.inputs.size()); - - for (uint32_t serialized_input_idx : node_entry.inputs) { - if (serialized_input_idx >= runtime_ids.size()) { - throw std::runtime_error( - "Graph file corrupted: input refers to a node that has not been reconstructed yet" - ); - } - runtime_inputs.push_back(runtime_ids[serialized_input_idx]); - } - - size_t new_node_id = 0; - - if (node_entry.op_type == OpType::INPUT) { - new_node_id = graph.input(node_entry.output_shape, node_entry.precision); - } - else { - OpParams params = node_entry.params; - params.output_precision = node_entry.precision; - populate_derived_params(graph, node_entry, runtime_inputs, params); - new_node_id = graph.add_node(node_entry.op_type, runtime_inputs, node_entry.output_shape, params); - - if (node_entry.op_type == OpType::PERSISTENT) { - graph.persistent_node_ids_.insert(new_node_id); - } - } - runtime_ids.push_back(new_node_id); - } - return graph; -} - -CactusGraph CactusGraph::load(const std::string& path) { - GraphFile::SerializedGraph sg = GraphFile::load_graph(path); - return from_serialized(sg); -} - -size_t CactusGraph::mmap_embeddings(const std::string& filename) { - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - if (shape.size() != 2) { - throw std::runtime_error("Memory-mapped embeddings must be 2D [vocab_size, embedding_dim]"); - } - - Precision precision = mapped_file->precision(); - - size_t node_id = input(shape, precision); - set_external_input(node_id, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - size_t file_idx = mapped_files_.size(); - mapped_files_.push_back(std::move(mapped_file)); - node_to_mapped_file_[node_id] = file_idx; - weight_cache_[filename] = node_id; - return node_id; -} - -size_t CactusGraph::mmap_weights(const std::string& filename) { - auto it = weight_cache_.find(filename); - if (it != weight_cache_.end()) { - return it->second; - } - - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - Precision precision = mapped_file->precision(); - - size_t node_id = input(shape, precision); - set_external_input(node_id, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(node_id, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(node_id)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - size_t file_idx = mapped_files_.size(); - mapped_files_.push_back(std::move(mapped_file)); - node_to_mapped_file_[node_id] = file_idx; - weight_cache_[filename] = node_id; - return node_id; -} - -void CactusGraph::release_weight_pages(size_t node_id) { - auto it = node_to_mapped_file_.find(node_id); - if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { - mapped_files_[it->second]->release_pages(); - } -} - -void CactusGraph::prefetch_weight_pages(size_t node_id) { - auto it = node_to_mapped_file_.find(node_id); - if (it != node_to_mapped_file_.end() && it->second < mapped_files_.size()) { - mapped_files_[it->second]->prefetch_pages(); - } -} - -void CactusGraph::release_all_weight_pages() { - for (auto& mf : mapped_files_) { - if (mf) mf->release_pages(); - } -} - -void CactusGraph::set_grouped_scales(size_t node_id, size_t group_size, size_t num_groups, void* scales_ptr) { - auto it = node_index_map_.find(node_id); - if (it != node_index_map_.end()) { - nodes_[it->second]->output_buffer.set_grouped_scales(group_size, num_groups, scales_ptr); - } -} - -void CactusGraph::set_interleaved(size_t node_id, bool interleaved, size_t original_N) { - auto it = node_index_map_.find(node_id); - if (it != node_index_map_.end()) { - nodes_[it->second]->output_buffer.set_interleaved(interleaved, original_N); - } -} - -size_t CactusGraph::embedding(const std::string& filename, size_t indices) { - auto mapped_file = std::make_unique(filename); - - const auto& shape = mapped_file->shape(); - if (shape.size() != 2) { - throw std::runtime_error("Embedding file must contain 2D tensor [vocab_size, hidden_dim]"); - } - - Precision precision = mapped_file->precision(); - size_t embeddings_node = input(shape, precision); - set_external_input(embeddings_node, const_cast(mapped_file->data()), precision); - - if (PrecisionTraits::is_integer(precision) && mapped_file->group_size() > 0) { - set_grouped_scales(embeddings_node, mapped_file->group_size(), mapped_file->num_groups(), - const_cast(mapped_file->scales_data())); - - if (mapped_file->is_interleaved()) { - auto& buffer = nodes_[node_index_map_.at(embeddings_node)]->output_buffer; - buffer.set_interleaved(true, mapped_file->original_N()); - } - } - - mapped_files_.push_back(std::move(mapped_file)); - - const auto& idx_shape = get_output_buffer(indices).shape; - std::vector output_shape = idx_shape; - output_shape.push_back(shape[1]); - - OpParams params; - params.output_precision = Precision::FP16; - return add_node(OpType::EMBEDDING, {embeddings_node, indices}, output_shape, params); -} - - -namespace GraphFile { - -void save_graph(const CactusGraph& graph, - const std::string& filename) { - - std::ofstream out(filename, std::ios::binary); - if (!out) { - throw std::runtime_error("Cannot open file for writing: " + filename); - } - - std::unordered_map id_to_index; - id_to_index.reserve(graph.nodes_.size()); - - for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { - id_to_index[graph.nodes_[i]->id] = i; - } - - SerializedGraph sg; - sg.header.magic = CACTUS_GRAPH_MAGIC; - sg.header.version = 4; - sg.header.node_count = static_cast(graph.nodes_.size()); - sg.header.flags = 0; - - sg.nodes.reserve(graph.nodes_.size()); - - for (uint32_t i = 0; i < graph.nodes_.size(); ++i) { - const auto& node = graph.nodes_[i]; - - NodeEntry entry; - entry.index = i; - entry.op_type = node->op_type; - entry.output_shape = node->output_buffer.shape; - entry.precision = node->output_buffer.precision; - entry.params = node->params; - - entry.inputs.reserve(node->input_ids.size()); - for (size_t input_id : node->input_ids) { - auto it = id_to_index.find(input_id); - if (it == id_to_index.end()) { - throw std::runtime_error("Graph save failed: missing input id mapping"); - } - entry.inputs.push_back(it->second); - } - - if (node->op_type == OpType::INPUT) { - sg.graph_inputs.push_back(entry.index); - } - - sg.nodes.push_back(std::move(entry)); - } - - sg.graph_outputs = compute_leaf_outputs(sg); - - write_serialized_graph(out, sg); - - if (!out) { - throw std::runtime_error("Error writing graph data to file: " + filename); - } -} - -SerializedGraph load_graph(const std::string& filename) { - std::ifstream in(filename, std::ios::binary); - if (!in) { - throw std::runtime_error("Cannot open file for reading: " + filename); - } - SerializedGraph sg; - sg.header = read_graph_header(in); - sg.graph_inputs = read_u32_vector(in); - sg.graph_outputs = read_u32_vector(in); - sg.nodes.reserve(sg.header.node_count); - for (uint32_t i = 0; i < sg.header.node_count; ++i) { - sg.nodes.push_back(read_node_entry(in)); - } - - if (sg.nodes.size() != sg.header.node_count) { - throw std::runtime_error("Graph file corrupted: node count mismatch"); - } - - for (uint32_t i = 0; i < sg.nodes.size(); ++i) { - const auto& node = sg.nodes[i]; - - if (node.index != i) { - throw std::runtime_error("Graph file corrupted: node indices must be dense and ordered"); - } - - for (uint32_t input_idx : node.inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: input index out of range"); - } - } - } - - for (uint32_t input_idx : sg.graph_inputs) { - if (input_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph input index out of range"); - } - } - - for (uint32_t output_idx : sg.graph_outputs) { - if (output_idx >= sg.nodes.size()) { - throw std::runtime_error("Graph file corrupted: graph output index out of range"); - } - } - - return sg; -} - -void save_node(CactusGraph& graph, size_t node_id, const std::string& filename) { - graph.execute(); - void* data = graph.get_output(node_id); - - const auto& buffer = graph.get_output_buffer(node_id); - const auto& shape = buffer.shape; - Precision precision = buffer.precision; - - std::ofstream file(filename, std::ios::binary); - if (!file) { - throw std::runtime_error("Cannot open file for writing: " + filename); - } - - size_t total_elements = 1; - for (size_t dim : shape) { - total_elements *= dim; - } - - size_t byte_size = PrecisionTraits::packed_size_of(precision, total_elements); - - bool has_scales = PrecisionTraits::is_integer(precision) && buffer.group_size > 0 && buffer.scales_data; - size_t N = shape.size() >= 1 ? shape[0] : 1; - size_t scales_bytes = has_scales ? (N * buffer.num_groups * sizeof(__fp16)) : 0; - - uint32_t ndim = static_cast(shape.size()); - uint32_t flags = has_scales ? FLAG_HAS_SCALES : 0; - if (buffer.is_interleaved) { - flags |= FLAG_INTERLEAVED; - } - uint32_t alignment = 32; - - uint32_t magic = CACTUS_MAGIC; - file.write(reinterpret_cast(&magic), sizeof(magic)); - file.write(reinterpret_cast(&flags), sizeof(flags)); - file.write(reinterpret_cast(&alignment), sizeof(alignment)); - file.write(reinterpret_cast(&ndim), sizeof(ndim)); - - for (uint32_t i = 0; i < 4; i++) { - uint64_t dim_val = (i < shape.size()) ? static_cast(shape[i]) : 0; - file.write(reinterpret_cast(&dim_val), sizeof(dim_val)); - } - - uint32_t prec_val = static_cast(precision); - file.write(reinterpret_cast(&prec_val), sizeof(prec_val)); - - uint64_t data_bytes = static_cast(byte_size); - uint64_t scales_bytes_val = static_cast(scales_bytes); - file.write(reinterpret_cast(&data_bytes), sizeof(data_bytes)); - file.write(reinterpret_cast(&scales_bytes_val), sizeof(scales_bytes_val)); - - uint32_t group_size = has_scales ? static_cast(buffer.group_size) : 0; - uint32_t num_groups = has_scales ? static_cast(buffer.num_groups) : 0; - file.write(reinterpret_cast(&group_size), sizeof(group_size)); - file.write(reinterpret_cast(&num_groups), sizeof(num_groups)); - - uint64_t original_N = buffer.is_interleaved ? buffer.original_N : N; - file.write(reinterpret_cast(&original_N), sizeof(original_N)); - - size_t header_end = HEADER_SIZE; - size_t aligned_header = align_offset(header_end, alignment); - size_t header_padding = aligned_header - header_end; - for (size_t i = 0; i < header_padding; i++) { - char zero = 0; - file.write(&zero, 1); - } - - if (has_scales) { - file.write(static_cast(buffer.scales_data), scales_bytes); - - size_t scales_end = aligned_header + scales_bytes; - size_t data_start = align_offset(scales_end, alignment); - size_t scales_padding = data_start - scales_end; - for (size_t i = 0; i < scales_padding; i++) { - char zero = 0; - file.write(&zero, 1); - } - } - - file.write(static_cast(data), byte_size); - - if (!file) { - throw std::runtime_error("Error writing node data to file: " + filename); - } -} - -// MappedFile implementation - -MappedFile::MappedFile(const std::string& filename) - : fd_(-1), mapped_data_(nullptr), file_size_(0), data_offset_(0) { - fd_ = open(filename.c_str(), O_RDONLY); - if (fd_ == -1) { - throw std::runtime_error("Cannot open file for mapping: " + filename); - } - - struct stat st; - if (fstat(fd_, &st) == -1) { - close(fd_); - throw std::runtime_error("Cannot get file size: " + filename); - } - file_size_ = static_cast(st.st_size); - - mapped_data_ = mmap(nullptr, file_size_, PROT_READ, MAP_SHARED, fd_, 0); - if (mapped_data_ == MAP_FAILED) { - close(fd_); - throw std::runtime_error("Cannot map file: " + filename); - } - - close(fd_); - fd_ = -1; - - parse_header(); - apply_madvise_hints(); -} - -MappedFile::~MappedFile() { - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - madvise(mapped_data_, file_size_, MADV_DONTNEED); - munmap(mapped_data_, file_size_); - mapped_data_ = nullptr; - } - if (fd_ != -1) { - close(fd_); - fd_ = -1; - } -} - -MappedFile::MappedFile(MappedFile&& other) noexcept - : fd_(other.fd_), mapped_data_(other.mapped_data_), file_size_(other.file_size_), - data_offset_(other.data_offset_), shape_(std::move(other.shape_)), - precision_(other.precision_), byte_size_(other.byte_size_), - group_size_(other.group_size_), num_groups_(other.num_groups_), - scales_offset_(other.scales_offset_), scales_bytes_(other.scales_bytes_), - alignment_(other.alignment_), - is_interleaved_(other.is_interleaved_), - original_N_(other.original_N_) { - other.fd_ = -1; - other.mapped_data_ = nullptr; - other.file_size_ = 0; - other.is_interleaved_ = false; - other.original_N_ = 0; -} - -MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { - if (this != &other) { - if (mapped_data_ != nullptr && mapped_data_ != MAP_FAILED) { - munmap(mapped_data_, file_size_); - } - if (fd_ != -1) { - close(fd_); - } - - fd_ = other.fd_; - mapped_data_ = other.mapped_data_; - file_size_ = other.file_size_; - data_offset_ = other.data_offset_; - shape_ = std::move(other.shape_); - precision_ = other.precision_; - byte_size_ = other.byte_size_; - group_size_ = other.group_size_; - num_groups_ = other.num_groups_; - scales_offset_ = other.scales_offset_; - scales_bytes_ = other.scales_bytes_; - alignment_ = other.alignment_; - is_interleaved_ = other.is_interleaved_; - original_N_ = other.original_N_; - other.fd_ = -1; - other.mapped_data_ = nullptr; - other.file_size_ = 0; - other.is_interleaved_ = false; - other.original_N_ = 0; - } - return *this; -} - -const std::vector& MappedFile::shape() const { - return shape_; -} - -Precision MappedFile::precision() const { - return precision_; -} - -size_t MappedFile::byte_size() const { - return byte_size_; -} - -const void* MappedFile::scales_data() const { - return static_cast(mapped_data_) + scales_offset_; -} - -void* MappedFile::data() { - return static_cast(mapped_data_) + data_offset_; -} - -const void* MappedFile::data() const { - return static_cast(mapped_data_) + data_offset_; -} - -template -const T* MappedFile::typed_data() const { - return static_cast(data()); -} - -void MappedFile::parse_header() { - if (file_size_ < HEADER_SIZE) { - throw std::runtime_error("File too small: insufficient data for header"); - } - - const char* ptr = static_cast(mapped_data_); - size_t offset = 0; - - uint32_t magic = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - if (magic != CACTUS_MAGIC) { - throw std::runtime_error("Invalid tensor file: missing CACT magic number"); - } - - uint32_t flags = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - is_interleaved_ = (flags & FLAG_INTERLEAVED) != 0; - - alignment_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - if (alignment_ == 0) alignment_ = 1; - - uint32_t ndim = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - shape_.clear(); - for (uint32_t i = 0; i < 4; i++) { - uint64_t dim_val = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - if (i < ndim && dim_val > 0) { - shape_.push_back(static_cast(dim_val)); - } - } - - uint32_t prec_val = *reinterpret_cast(ptr + offset); - precision_ = static_cast(prec_val); - offset += sizeof(uint32_t); - - byte_size_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - scales_bytes_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - group_size_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - num_groups_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint32_t); - - original_N_ = *reinterpret_cast(ptr + offset); - offset += sizeof(uint64_t); - - size_t aligned_header = align_offset(HEADER_SIZE, alignment_); - - if (scales_bytes_ > 0) { - scales_offset_ = aligned_header; - size_t scales_end = scales_offset_ + scales_bytes_; - data_offset_ = align_offset(scales_end, alignment_); - } else { - scales_offset_ = 0; - data_offset_ = aligned_header; - } - - if (data_offset_ + byte_size_ > file_size_) { - throw std::runtime_error("File corrupted: data extends beyond file size"); - } - -} - -void MappedFile::apply_madvise_hints() { - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); - } - - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_SEQUENTIAL); - - if (byte_size_ > 1024 * 1024) { - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); - } -} - -void MappedFile::release_pages() { - if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; - - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_DONTNEED); - } - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_DONTNEED); -} - -void MappedFile::prefetch_pages() { - if (mapped_data_ == nullptr || mapped_data_ == MAP_FAILED) return; - - if (scales_bytes_ > 0 && scales_offset_ > 0) { - madvise(static_cast(mapped_data_) + scales_offset_, scales_bytes_, MADV_WILLNEED); - } - madvise(static_cast(mapped_data_) + data_offset_, byte_size_, MADV_WILLNEED); -} - -template const int8_t* MappedFile::typed_data() const; -template const float* MappedFile::typed_data() const; -template const uint16_t* MappedFile::typed_data() const; -template const uint8_t* MappedFile::typed_data() const; - -} // namespace GraphFile diff --git a/cactus/graph/graph_ops_math.cpp b/cactus/graph/graph_ops_math.cpp deleted file mode 100644 index 811da3511..000000000 --- a/cactus/graph/graph_ops_math.cpp +++ /dev/null @@ -1,348 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include -#include - -namespace Quantization { - void int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { - cactus_int8_to_fp32(src, dst, count, scale); - } - - void fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { - cactus_fp32_to_int8(src, dst, count, scale); - } - - void fp16_to_fp32(const __fp16* src, float* dst, size_t count) { - cactus_fp16_to_fp32(src, dst, count); - } - - void fp32_to_fp16(const float* src, __fp16* dst, size_t count) { - cactus_fp32_to_fp16(src, dst, count); - } - - void int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { - cactus_int8_to_fp16(src, dst, count, scale); - } - - void fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { - cactus_fp16_to_int8(src, dst, count, scale); - } -} - -static std::vector compute_strides(const std::vector& shape, const std::vector& target_shape) { - std::vector strides(target_shape.size()); - - size_t shape_offset = target_shape.size() - shape.size(); - - for (size_t i = 0; i < target_shape.size(); ++i) { - if (i < shape_offset) { - strides[i] = 0; - } else { - size_t dim_idx = i - shape_offset; - if (shape[dim_idx] == 1) { - strides[i] = 0; - } else { - strides[i] = 1; - for (size_t j = dim_idx + 1; j < shape.size(); ++j) { - strides[i] *= shape[j]; - } - } - } - } - - return strides; -} - -void dispatch_binary_op_f16(OpType op, const __fp16* lhs, const __fp16* rhs, __fp16* output, size_t count) { - switch (op) { - case OpType::ADD: - cactus_add_f16(lhs, rhs, output, count); - break; - case OpType::ADD_CLIPPED: - cactus_add_f16_clipped(lhs, rhs, output, count); - break; - case OpType::SUBTRACT: - cactus_subtract_f16(lhs, rhs, output, count); - break; - case OpType::MULTIPLY: - cactus_multiply_f16(lhs, rhs, output, count); - break; - case OpType::DIVIDE: - cactus_divide_f16(lhs, rhs, output, count); - break; - default: - break; - } -} - -void dispatch_unary_op_f16(OpType op, const __fp16* input, __fp16* output, size_t count, float param) { - ScalarOpType scalar_op; - switch (op) { - case OpType::SCALAR_ADD: scalar_op = ScalarOpType::ADD; break; - case OpType::SCALAR_SUBTRACT: scalar_op = ScalarOpType::SUBTRACT; break; - case OpType::SCALAR_MULTIPLY: scalar_op = ScalarOpType::MULTIPLY; break; - case OpType::SCALAR_DIVIDE: scalar_op = ScalarOpType::DIVIDE; break; - case OpType::SCALAR_EXP: scalar_op = ScalarOpType::EXP; break; - case OpType::SCALAR_SQRT: scalar_op = ScalarOpType::SQRT; break; - case OpType::SCALAR_COS: scalar_op = ScalarOpType::COS; break; - case OpType::SCALAR_SIN: scalar_op = ScalarOpType::SIN; break; - case OpType::SCALAR_LOG: scalar_op = ScalarOpType::LOG; break; - case OpType::ABS: scalar_op = ScalarOpType::ABS; break; - case OpType::POW: scalar_op = ScalarOpType::POW; break; - default: return; - } - - cactus_scalar_op_f16(input, output, count, param, scalar_op); -} - -void compute_binary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& lhs = get_input(node, 0, nodes, node_index_map); - const auto& rhs = get_input(node, 1, nodes, node_index_map); - - if (lhs.precision != Precision::FP16) { - throw std::runtime_error("Binary operations only support FP16 precision (got " + std::to_string(static_cast(lhs.precision)) + ")"); - } - - if (node.params.broadcast_info.needs_broadcasting) { - std::vector lhs_strides = compute_strides(lhs.shape, node.params.broadcast_info.output_shape); - std::vector rhs_strides = compute_strides(rhs.shape, node.params.broadcast_info.output_shape); - - switch (node.op_type) { - case OpType::ADD: - case OpType::ADD_CLIPPED: - cactus_add_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::SUBTRACT: - cactus_subtract_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::MULTIPLY: - cactus_multiply_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - case OpType::DIVIDE: - cactus_divide_broadcast_f16(lhs.data_as<__fp16>(), rhs.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - lhs_strides.data(), rhs_strides.data(), - node.params.broadcast_info.output_shape.data(), - node.params.broadcast_info.output_shape.size()); - break; - default: break; - } - } else { - dispatch_binary_op_f16(node.op_type, lhs.data_as<__fp16>(), - rhs.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - } -} - -void compute_unary_op_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - - if (input.precision != Precision::FP16) { - throw std::runtime_error("Scalar operations only support FP16 precision"); - } - - dispatch_unary_op_f16(node.op_type, input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size, node.params.scalar); -} - -void compute_activation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - - if (input.precision != Precision::FP16) { - throw std::runtime_error("Activation operations only support FP16 precision"); - } - - switch (node.op_type) { - case OpType::RELU: - cactus_relu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::SILU: - cactus_silu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::GELU: - cactus_gelu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::GELU_ERF: - cactus_gelu_f16_erf(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::SIGMOID: - cactus_sigmoid_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::TANH: - cactus_tanh_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size); - break; - case OpType::LEAKY_RELU: - cactus_leaky_relu_f16(input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - node.output_buffer.total_size, node.params.scalar); - break; - default: - break; - } -} - -void compute_reduce_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - int axis = node.params.axis; - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("Reduction operations only support FP16 precision"); - } - - if (axis == -1) { - switch (node.op_type) { - case OpType::SUM: { - double result = cactus_sum_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::MEAN: { - double result = cactus_mean_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::VARIANCE: { - double result = cactus_variance_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = static_cast<__fp16>(result); - break; - } - case OpType::MIN: { - __fp16 result = cactus_min_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = result; - break; - } - case OpType::MAX: { - __fp16 result = cactus_max_all_f16(input_buffer.data_as<__fp16>(), input_buffer.total_size); - node.output_buffer.data_as<__fp16>()[0] = result; - break; - } - default: break; - } - } else { - auto dims = AxisDims::from_shape(input_buffer.shape, static_cast(axis)); - - switch (node.op_type) { - case OpType::SUM: - cactus_sum_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MEAN: - cactus_mean_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::VARIANCE: - cactus_variance_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MIN: - cactus_min_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - case OpType::MAX: - cactus_max_axis_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - dims.outer, dims.axis_size, dims.inner); - break; - default: break; - } - } -} - -void compute_reshape_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - - size_t input_total_elements = input_buffer.total_size; - size_t output_total_elements = node.output_buffer.total_size; - - if (input_total_elements != output_total_elements) { - throw std::runtime_error("Reshape operation: input elements (" + std::to_string(input_total_elements) + - ") must match output elements (" + std::to_string(output_total_elements) + ")"); - } - - std::memcpy(node.output_buffer.get_data(), input_buffer.get_data(), input_buffer.byte_size); -} - -template -static void dequant_grouped_int8(const int8_t* src, OutT* dst, const __fp16* scales, - const std::vector& shape, size_t group_size) { - if (shape.size() == 2) { - size_t N = shape[0], K = shape[1]; - size_t num_groups = K / group_size; - for (size_t row = 0; row < N; ++row) { - for (size_t col = 0; col < K; ++col) { - size_t idx = row * K + col; - float scale = static_cast(scales[row * num_groups + col / group_size]); - dst[idx] = static_cast(static_cast(src[idx]) * scale); - } - } - } else if (shape.size() == 1) { - size_t K = shape[0]; - for (size_t col = 0; col < K; ++col) { - float scale = static_cast(scales[col / group_size]); - dst[col] = static_cast(static_cast(src[col]) * scale); - } - } -} - -void compute_precision_cast_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buf = get_input(node, 0, nodes, node_index_map); - - if (input_buf.precision == node.output_buffer.precision) { - std::memcpy(node.output_buffer.get_data(), input_buf.get_data(), input_buf.byte_size); - return; - } - - size_t count = input_buf.total_size; - - if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP32) { - if (input_buf.is_grouped_int8()) { - dequant_grouped_int8(input_buf.data_as(), node.output_buffer.data_as(), - input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); - } else { - Quantization::int8_to_fp32(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); - } - } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::INT8) { - Quantization::fp32_to_int8(input_buf.data_as(), node.output_buffer.data_as(), count, 1.0f); - } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::FP32) { - Quantization::fp16_to_fp32(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count); - } else if (input_buf.precision == Precision::FP32 && node.output_buffer.precision == Precision::FP16) { - Quantization::fp32_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count); - } else if (input_buf.precision == Precision::INT8 && node.output_buffer.precision == Precision::FP16) { - if (input_buf.is_grouped_int8()) { - dequant_grouped_int8<__fp16>(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), - input_buf.scales_as_fp16(), input_buf.shape, input_buf.group_size); - } else { - Quantization::int8_to_fp16(input_buf.data_as(), node.output_buffer.data_as<__fp16>(), count, 1.0f); - } - } else if (input_buf.precision == Precision::FP16 && node.output_buffer.precision == Precision::INT8) { - Quantization::fp16_to_int8(input_buf.data_as<__fp16>(), node.output_buffer.data_as(), count, 1.0f); - } else { - throw std::runtime_error("Unsupported precision conversion from " + - std::to_string(static_cast(input_buf.precision)) + - " to " + std::to_string(static_cast(node.output_buffer.precision))); - } -} diff --git a/cactus/graph/graph_ops_nn.cpp b/cactus/graph/graph_ops_nn.cpp deleted file mode 100644 index ae6f92b0e..000000000 --- a/cactus/graph/graph_ops_nn.cpp +++ /dev/null @@ -1,2532 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - thread_local std::vector<__fp16> transpose_buffer_fp16; - thread_local std::vector quant_activation_buffer; - thread_local std::vector quant_scales_buffer; - - thread_local const __fp16* cached_quant_src = nullptr; - thread_local size_t cached_quant_M = 0; - thread_local size_t cached_quant_K = 0; - - void ensure_transpose_buffer_fp16(size_t required_size) { - if (transpose_buffer_fp16.size() < required_size) { - transpose_buffer_fp16.resize(required_size); - } - } - - void ensure_quant_buffers(size_t M, size_t K) { - size_t required_data = M * K; - if (quant_activation_buffer.size() < required_data) { - quant_activation_buffer.resize(required_data); - } - if (quant_scales_buffer.size() < M) { - quant_scales_buffer.resize(M); - } - } - - void quantize_activations_fp16_to_int8(const __fp16* src, int8_t* dst, float* scales, size_t M, size_t K) { - if (src == cached_quant_src && M == cached_quant_M && K == cached_quant_K) { - return; - } - - constexpr size_t PARALLEL_THRESHOLD = 16; - - if (M >= PARALLEL_THRESHOLD) { - CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, K](size_t m_start, size_t m_end) { - for (size_t m = m_start; m < m_end; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - }); - } else { - for (size_t m = 0; m < M; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - } - - cached_quant_src = src; - cached_quant_M = M; - cached_quant_K = K; - } - - const __fp16* as_fp16_ptr(const BufferDesc& buffer, std::vector<__fp16>& scratch) { - if (buffer.precision == Precision::FP16) { - return buffer.data_as<__fp16>(); - } - if (buffer.precision == Precision::FP32) { - scratch.resize(buffer.total_size); - cactus_fp32_to_fp16(buffer.data_as(), scratch.data(), buffer.total_size); - return scratch.data(); - } - throw std::runtime_error("GATED_DELTANET unsupported precision (expected FP16/FP32)"); - } - - void validate_gated_deltanet_inputs( - const BufferDesc& q, - const BufferDesc& k, - const BufferDesc& v, - const BufferDesc& g, - const BufferDesc& b, - const BufferDesc& s) { - auto is_supported_precision = [](Precision p) { - return p == Precision::FP16 || p == Precision::FP32; - }; - if (!is_supported_precision(q.precision) || !is_supported_precision(k.precision) || - !is_supported_precision(v.precision) || !is_supported_precision(g.precision) || - !is_supported_precision(b.precision) || !is_supported_precision(s.precision)) { - throw std::runtime_error("GATED_DELTANET requires FP16/FP32 inputs"); - } - - if (q.shape.size() != 4 || k.shape.size() != 4 || v.shape.size() != 4) { - throw std::runtime_error("GATED_DELTANET expects query/key/value rank 4 [B, T, H, D]"); - } - if (g.shape.size() != 3 || b.shape.size() != 3) { - throw std::runtime_error("GATED_DELTANET expects gate_log/beta rank 3 [B, T, H]"); - } - if (s.shape.size() != 4) { - throw std::runtime_error("GATED_DELTANET expects state rank 4 [B, K, H, V]"); - } - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - - if (k.shape[0] != B || k.shape[1] != T || k.shape[2] != Hq || k.shape[3] != K) { - throw std::runtime_error("GATED_DELTANET query/key shape mismatch"); - } - if (v.shape[0] != B || v.shape[1] != T) { - throw std::runtime_error("GATED_DELTANET value shape mismatch"); - } - const size_t Hv = v.shape[2]; - if (g.shape[0] != B || g.shape[1] != T || g.shape[2] != Hv || - b.shape[0] != B || b.shape[1] != T || b.shape[2] != Hv) { - throw std::runtime_error("GATED_DELTANET gate_log/beta shape mismatch"); - } - if (Hq == 0 || Hv == 0 || (Hv % Hq) != 0) { - throw std::runtime_error("GATED_DELTANET expects value heads divisible by q/k heads"); - } - const size_t V = v.shape[3]; - if (s.shape[0] != B || s.shape[1] != K || s.shape[2] != Hv || s.shape[3] != V) { - throw std::runtime_error("GATED_DELTANET state shape mismatch"); - } - } - - -} - -void shrink_thread_local_buffers() { - std::vector<__fp16>().swap(transpose_buffer_fp16); - std::vector().swap(quant_activation_buffer); - std::vector().swap(quant_scales_buffer); - cached_quant_src = nullptr; - cached_quant_M = 0; - cached_quant_K = 0; -} - -void compute_quantize_activations_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("QUANTIZE_ACTIVATIONS requires FP16 input"); - } - - if (shape.size() < 2) { - throw std::runtime_error("QUANTIZE_ACTIVATIONS requires at least 2D tensor"); - } - - size_t K = shape.back(); - size_t M = 1; - for (size_t i = 0; i < shape.size() - 1; ++i) { - M *= shape[i]; - } - - if (!node.output_buffer.has_activation_scales() || - node.output_buffer.num_rows_for_activation_scales != M) { - node.output_buffer.allocate_activation_scales(M); - } - - const __fp16* src = input_buffer.data_as<__fp16>(); - int8_t* dst = node.output_buffer.data_as(); - float* scales = node.output_buffer.activation_scales_as_float(); - - constexpr size_t PARALLEL_THRESHOLD = 16; - - if (M >= PARALLEL_THRESHOLD) { - CactusThreading::parallel_for(M, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, K](size_t m_start, size_t m_end) { - for (size_t m = m_start; m < m_end; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - }); - } else { - for (size_t m = 0; m < M; m++) { - float max_abs = cactus_fp16_max_abs(src + m * K, K); - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - scales[m] = scale; - cactus_fp16_to_int8(src + m * K, dst + m * K, K, scale); - } - } -} - -void compute_matmul_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& lhs_buffer = get_input(node, 0, nodes, node_index_map); - const auto& rhs_buffer = get_input(node, 1, nodes, node_index_map); - const auto& lhs_shape = lhs_buffer.shape; - const auto& rhs_shape = rhs_buffer.shape; - - size_t M = lhs_shape[lhs_shape.size() - 2]; - size_t K = lhs_shape[lhs_shape.size() - 1]; - size_t N; - if (rhs_buffer.is_interleaved && rhs_buffer.original_N > 0) { - N = rhs_buffer.original_N; - } else { - N = node.params.pretransposed_rhs ? - rhs_shape[rhs_shape.size() - 2] : rhs_shape[rhs_shape.size() - 1]; - } - - bool pretransposed_rhs = node.params.pretransposed_rhs; - - ComputeBackend backend = node.params.backend; - - if (backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU matrix multiplication not yet implemented"); - } - - const bool lhs_is_prequantized_int8 = (lhs_buffer.precision == Precision::INT8 && - lhs_buffer.has_activation_scales()); - - if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { - const int8_t* rhs = rhs_buffer.data_as(); - const __fp16* rhs_scales = rhs_buffer.scales_as_fp16(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (!pretransposed_rhs) { - throw std::runtime_error("Group-wise quantized matmul requires pretransposed weights"); - } - - const int8_t* lhs_int8; - const float* lhs_scales; - - if (lhs_is_prequantized_int8) { - lhs_int8 = lhs_buffer.data_as(); - lhs_scales = lhs_buffer.activation_scales_as_float(); - } else if (lhs_buffer.precision == Precision::FP16) { - ensure_quant_buffers(M, K); - cached_quant_src = nullptr; - quantize_activations_fp16_to_int8(lhs_buffer.data_as<__fp16>(), quant_activation_buffer.data(), - quant_scales_buffer.data(), M, K); - lhs_int8 = quant_activation_buffer.data(); - lhs_scales = quant_scales_buffer.data(); - } else { - throw std::runtime_error("Quantized matmul requires INT8 (pre-quantized) or FP16 activations"); - } - - cactus_matmul_integer(rhs_buffer.precision, - lhs_int8, lhs_scales, - rhs, rhs_scales, output, - M, K, N, rhs_buffer.group_size); - } else { - if (lhs_buffer.precision != Precision::FP16) { - throw std::runtime_error("FP16 matmul requires FP16 activations (got precision " + std::to_string(static_cast(lhs_buffer.precision)) + ")"); - } - - const __fp16* lhs = lhs_buffer.data_as<__fp16>(); - const __fp16* rhs = rhs_buffer.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (pretransposed_rhs) { - cactus_matmul_f16(lhs, rhs, output, M, K, N); - } else { - size_t transpose_size = rhs_shape[0] * rhs_shape[1]; - ensure_transpose_buffer_fp16(transpose_size); - - cactus_transpose_2d_f16(rhs, transpose_buffer_fp16.data(), - rhs_shape[0], rhs_shape[1], 0, rhs_shape[0]); - cactus_matmul_f16(lhs, transpose_buffer_fp16.data(), output, M, K, N); - } - } -} - -namespace { - thread_local std::vector<__fp16> moe_compact_hidden_buf; - thread_local std::vector<__fp16> moe_gate_buf; - thread_local std::vector<__fp16> moe_up_buf; - thread_local std::vector<__fp16> moe_expert_out_buf; - thread_local std::vector moe_lhs_q_buf; - thread_local std::vector moe_lhs_scales_buf; - thread_local std::vector moe_expert_offsets_buf; - thread_local std::vector moe_expert_tokens_buf; - thread_local std::vector moe_routing_denom_buf; - - void ensure_moe_buffers(size_t max_tokens, size_t hidden_dim, size_t intermediate_dim, - size_t num_experts, size_t top_k) { - size_t hidden_size = max_tokens * hidden_dim; - size_t inter_size = max_tokens * intermediate_dim; - if (moe_compact_hidden_buf.size() < hidden_size) moe_compact_hidden_buf.resize(hidden_size); - if (moe_gate_buf.size() < inter_size) moe_gate_buf.resize(inter_size); - if (moe_up_buf.size() < inter_size) moe_up_buf.resize(inter_size); - if (moe_expert_out_buf.size() < hidden_size) moe_expert_out_buf.resize(hidden_size); - size_t max_k = std::max(hidden_dim, intermediate_dim); - size_t quant_size = max_tokens * max_k; - if (moe_lhs_q_buf.size() < quant_size) moe_lhs_q_buf.resize(quant_size); - if (moe_lhs_scales_buf.size() < max_tokens) moe_lhs_scales_buf.resize(max_tokens); - size_t total_assignments = max_tokens * top_k; - if (moe_expert_offsets_buf.size() < num_experts + 1) moe_expert_offsets_buf.resize(num_experts + 1); - if (moe_expert_tokens_buf.size() < total_assignments) moe_expert_tokens_buf.resize(total_assignments); - if (moe_routing_denom_buf.size() < max_tokens) moe_routing_denom_buf.resize(max_tokens); - } - - void moe_matmul(const __fp16* lhs, - size_t M, - size_t K, - const BufferDesc& rhs_buffer, - __fp16* output, - size_t N, - bool lhs_prequantized = false) { - if (rhs_buffer.precision == Precision::FP16) { - cactus_matmul_f16(lhs, rhs_buffer.data_as<__fp16>(), output, M, K, N); - return; - } - - if (PrecisionTraits::is_integer(rhs_buffer.precision) && rhs_buffer.group_size > 0) { - int8_t* lhs_q = moe_lhs_q_buf.data(); - float* lhs_scales = moe_lhs_scales_buf.data(); - if (!lhs_prequantized) { - for (size_t row = 0; row < M; ++row) { - float scale = cactus_fp16_max_abs(lhs + row * K, K) / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - lhs_scales[row] = scale; - cactus_fp16_to_int8(lhs + row * K, lhs_q + row * K, K, scale); - } - } - cactus_matmul_integer(rhs_buffer.precision, - lhs_q, lhs_scales, - rhs_buffer.data_as(), - rhs_buffer.scales_as_fp16(), - output, M, K, N, rhs_buffer.group_size); - return; - } - - throw std::runtime_error("moe_layer only supports FP16 or grouped INT4/INT8 expert weights"); - } -} - -void compute_moe_layer_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const size_t num_experts = node.params.num_experts; - const size_t top_k = node.params.num_experts_per_tok; - const bool normalize_routing = node.params.normalize_routing; - const float eps = node.params.epsilon; - const float routed_scaling_factor = node.params.scalar; - const bool gated = node.params.moe_gated; - const Activation activation = node.params.activation; - const size_t base_inputs = gated ? (3 + 3 * num_experts) : (3 + 2 * num_experts); - bool has_per_expert_scale = node.input_ids.size() == base_inputs + 1; - if (node.input_ids.size() != base_inputs && node.input_ids.size() != base_inputs + 1) { - throw std::runtime_error("moe_layer expects " + std::to_string(base_inputs) + " or " + std::to_string(base_inputs + 1) + " inputs, got " + std::to_string(node.input_ids.size())); - } - - const auto& hidden_buffer = get_input(node, 0, nodes, node_index_map); - const auto& routing_buffer = get_input(node, 1, nodes, node_index_map); - const auto& topk_idx_buffer = get_input(node, 2, nodes, node_index_map); - - if (hidden_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { - throw std::runtime_error("moe_layer expects FP16 hidden/output"); - } - if (topk_idx_buffer.precision != Precision::FP32) { - throw std::runtime_error("moe_layer expects FP32 topk indices"); - } - - const __fp16* expert_scales_fp16 = nullptr; - if (has_per_expert_scale) { - const auto& scale_buffer = get_input(node, base_inputs, nodes, node_index_map); - if (scale_buffer.precision != Precision::FP16) { - throw std::runtime_error("moe_layer expects FP16 per_expert_scale"); - } - expert_scales_fp16 = scale_buffer.data_as<__fp16>(); - } - - const size_t token_count = hidden_buffer.shape[0]; - const size_t hidden_dim = hidden_buffer.shape[1]; - const size_t total_num_experts = routing_buffer.shape[1]; - - const auto& w1_0_buffer = get_input(node, 3, nodes, node_index_map); - const size_t expert_intermediate_dim = w1_0_buffer.shape[0]; - - const auto* hidden = hidden_buffer.data_as<__fp16>(); - auto* output = node.output_buffer.data_as<__fp16>(); - const auto* topk_idx = topk_idx_buffer.data_as(); - const auto* routing_fp16 = routing_buffer.precision == Precision::FP16 ? routing_buffer.data_as<__fp16>() : nullptr; - const auto* routing_fp32 = routing_buffer.precision == Precision::FP32 ? routing_buffer.data_as() : nullptr; - - auto routing_prob = [&](size_t tok, size_t exp) -> float { - const size_t offset = tok * total_num_experts + exp; - if (routing_fp16) return static_cast(routing_fp16[offset]); - return routing_fp32[offset]; - }; - - ensure_moe_buffers(token_count, hidden_dim, expert_intermediate_dim, num_experts, top_k); - - size_t* expert_offsets = moe_expert_offsets_buf.data(); - size_t* expert_tokens_flat = moe_expert_tokens_buf.data(); - - std::memset(expert_offsets, 0, (num_experts + 1) * sizeof(size_t)); - for (size_t tok = 0; tok < token_count; ++tok) { - for (size_t k = 0; k < top_k; ++k) { - float raw_idx = topk_idx[tok * top_k + k]; - if (!std::isfinite(raw_idx)) { - throw std::runtime_error("moe_layer got non-finite expert index"); - } - size_t idx = static_cast(raw_idx + 0.5f); - if (idx >= num_experts) { - throw std::runtime_error("moe_layer got expert index out of range"); - } - expert_offsets[idx + 1]++; - } - } - - for (size_t e = 0; e < num_experts; ++e) { - expert_offsets[e + 1] += expert_offsets[e]; - } - - thread_local std::vector moe_write_cursors; - if (moe_write_cursors.size() < num_experts) moe_write_cursors.resize(num_experts); - std::memcpy(moe_write_cursors.data(), expert_offsets, num_experts * sizeof(size_t)); - - for (size_t tok = 0; tok < token_count; ++tok) { - for (size_t k = 0; k < top_k; ++k) { - size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); - expert_tokens_flat[moe_write_cursors[idx]++] = tok; - } - } - - float* routing_denom = moe_routing_denom_buf.data(); - if (normalize_routing) { - for (size_t tok = 0; tok < token_count; ++tok) { - float sum_probs = 0.0f; - for (size_t k = 0; k < top_k; ++k) { - size_t idx = static_cast(topk_idx[tok * top_k + k] + 0.5f); - sum_probs += routing_prob(tok, idx); - } - routing_denom[tok] = sum_probs + eps; - } - } - - std::memset(output, 0, token_count * hidden_dim * sizeof(__fp16)); - - for (size_t expert_idx = 0; expert_idx < num_experts; ++expert_idx) { - const size_t start = expert_offsets[expert_idx]; - const size_t end = expert_offsets[expert_idx + 1]; - if (start == end) continue; - - const size_t selected_count = end - start; - const size_t* selected_tokens = expert_tokens_flat + start; - - const auto& w1_buffer = get_input(node, 3 + expert_idx, nodes, node_index_map); - const auto& w2_buffer = gated - ? get_input(node, 3 + 2 * num_experts + expert_idx, nodes, node_index_map) - : get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); - - __fp16* compact_hidden = moe_compact_hidden_buf.data(); - for (size_t i = 0; i < selected_count; ++i) { - std::memcpy(compact_hidden + i * hidden_dim, - hidden + selected_tokens[i] * hidden_dim, - hidden_dim * sizeof(__fp16)); - } - - __fp16* gate = moe_gate_buf.data(); - __fp16* up = moe_up_buf.data(); - __fp16* expert_out = moe_expert_out_buf.data(); - - moe_matmul(compact_hidden, selected_count, hidden_dim, w1_buffer, gate, expert_intermediate_dim); - const bool w1_was_int8 = w1_buffer.is_grouped_int8(); - - switch (activation) { - case Activation::GELU: - cactus_gelu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::GELU_ERF: - cactus_gelu_f16_erf(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::RELU: - cactus_relu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - case Activation::SILU: - default: - cactus_silu_f16(gate, gate, selected_count * expert_intermediate_dim); - break; - } - - if (gated) { - const auto& w3_buffer = get_input(node, 3 + num_experts + expert_idx, nodes, node_index_map); - moe_matmul(compact_hidden, selected_count, hidden_dim, w3_buffer, up, expert_intermediate_dim, w1_was_int8); - cactus_multiply_f16(gate, up, gate, selected_count * expert_intermediate_dim); - } - - moe_matmul(gate, selected_count, expert_intermediate_dim, w2_buffer, expert_out, hidden_dim); - - for (size_t i = 0; i < selected_count; ++i) { - const size_t tok = selected_tokens[i]; - float expert_prob = routing_prob(tok, expert_idx); - if (expert_prob <= 0.0f) continue; - - float route_weight = expert_prob; - if (normalize_routing) { - route_weight = expert_prob / routing_denom[tok]; - } - route_weight *= routed_scaling_factor; - if (expert_scales_fp16) { - route_weight *= static_cast(expert_scales_fp16[expert_idx]); - } - - auto* out_row = output + tok * hidden_dim; - const auto* expert_row = expert_out + i * hidden_dim; - cactus_add_scaled_f16(out_row, expert_row, out_row, hidden_dim, route_weight); - } - } -} - -namespace { - struct DenseFusedScratch { - std::vector<__fp16> gate_tile; - std::vector<__fp16> up_tile; - }; - - // Inline to skip cactus_fp16_to_int8's parallel_for orchestration, which - // dispatches ~5 threads at d_ffn=12288 and dominates the SIMD work. - static inline void fp16_to_int8_inplace(const __fp16* src, int8_t* dst, size_t n, float scale) { - const float inv_scale = 1.0f / scale; - const float32x4_t inv_vec = vdupq_n_f32(inv_scale); - const float32x4_t min_vec = vdupq_n_f32(-128.0f); - const float32x4_t max_vec = vdupq_n_f32(127.0f); - size_t i = 0; - for (; i + 8 <= n; i += 8) { - float16x8_t in = vld1q_f16(src + i); - float32x4_t lo = vmulq_f32(vcvt_f32_f16(vget_low_f16(in)), inv_vec); - float32x4_t hi = vmulq_f32(vcvt_f32_f16(vget_high_f16(in)), inv_vec); - lo = vmaxq_f32(vminq_f32(lo, max_vec), min_vec); - hi = vmaxq_f32(vminq_f32(hi, max_vec), min_vec); - int16x8_t s = vcombine_s16(vqmovn_s32(vcvtnq_s32_f32(lo)), - vqmovn_s32(vcvtnq_s32_f32(hi))); - vst1_s8(dst + i, vqmovn_s16(s)); - } - for (; i < n; ++i) { - float v = static_cast(src[i]) * inv_scale; - if (v > 127.0f) v = 127.0f; else if (v < -128.0f) v = -128.0f; - dst[i] = static_cast(std::lround(v)); - } - } -} - -void compute_dense_mlp_int4_fused_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& hidden_buf = get_input(node, 0, nodes, node_index_map); - const auto& gate_buf = get_input(node, 1, nodes, node_index_map); - const auto& up_buf = get_input(node, 2, nodes, node_index_map); - const auto& down_buf = get_input(node, 3, nodes, node_index_map); - - if (hidden_buf.precision != Precision::FP16) { - throw std::runtime_error("dense_mlp_int4_fused: hidden must be FP16"); - } - if (gate_buf.precision != Precision::INT4 || gate_buf.group_size == 0 || - up_buf.precision != Precision::INT4 || up_buf.group_size == 0 || - down_buf.precision != Precision::INT4 || down_buf.group_size == 0) { - throw std::runtime_error("dense_mlp_int4_fused: weights must be INT4 group-quantized"); - } - const size_t group_size = gate_buf.group_size; - if (up_buf.group_size != group_size || down_buf.group_size != group_size) { - throw std::runtime_error("dense_mlp_int4_fused: gate/up/down group_size must match"); - } - - const auto& shape = hidden_buf.shape; - const size_t hidden_dim_in = shape.back(); - size_t M = 1; - for (size_t i = 0; i + 1 < shape.size(); ++i) M *= shape[i]; - - const size_t d_ffn = gate_buf.is_interleaved ? gate_buf.original_N : gate_buf.shape[0]; - const size_t hidden_dim = down_buf.is_interleaved ? down_buf.original_N : down_buf.shape[0]; - if (hidden_dim_in != hidden_dim) { - throw std::runtime_error("dense_mlp_int4_fused: hidden dim mismatch with down weight"); - } - if (d_ffn % 4 != 0 || d_ffn % group_size != 0 || hidden_dim % 4 != 0) { - throw std::runtime_error("dense_mlp_int4_fused: d_ffn must be a multiple of 4 and of group_size, hidden_dim a multiple of 4"); - } - const size_t blocks_d_ffn = d_ffn / 4; - const size_t blocks_hidden = hidden_dim / 4; - - constexpr size_t tile_n_blocks = 32; - constexpr size_t tile_rows = tile_n_blocks * 4; - if (tile_rows % group_size != 0) { - throw std::runtime_error("dense_mlp_int4_fused: tile_rows must be a multiple of group_size"); - } - - const __fp16* hidden_fp16 = hidden_buf.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - const int8_t* gate_w = gate_buf.data_as(); - const __fp16* gate_s = gate_buf.scales_as_fp16(); - const int8_t* up_w = up_buf.data_as(); - const __fp16* up_s = up_buf.scales_as_fp16(); - const int8_t* down_w = down_buf.data_as(); - const __fp16* down_s = down_buf.scales_as_fp16(); - - thread_local std::vector dense_x_int8; - thread_local std::vector dense_x_scales; - thread_local std::vector<__fp16> dense_h_fp16; - thread_local std::vector dense_h_int8; - if (dense_x_int8.size() < M * hidden_dim) dense_x_int8.resize(M * hidden_dim); - if (dense_x_scales.size() < M) dense_x_scales.resize(M); - if (dense_h_fp16.size() < d_ffn) dense_h_fp16.resize(d_ffn); - if (dense_h_int8.size() < d_ffn) dense_h_int8.resize(d_ffn); - - auto& pool = CactusThreading::get_thread_pool(); - const size_t num_workers = pool.num_workers(); - std::vector dense_fused_scratch(num_workers); - for (auto& sc : dense_fused_scratch) { - sc.gate_tile.resize(tile_rows); - sc.up_tile.resize(tile_rows); - } - - for (size_t t = 0; t < M; ++t) { - const __fp16* x = hidden_fp16 + t * hidden_dim; - float maxabs = cactus_fp16_max_abs(x, hidden_dim); - float xs = std::max(maxabs / 127.0f, 1e-10f); - dense_x_scales[t] = xs; - cactus_fp16_to_int8(x, dense_x_int8.data() + t * hidden_dim, hidden_dim, xs); - } - - const size_t tiles_d_ffn = (blocks_d_ffn + tile_n_blocks - 1) / tile_n_blocks; - const size_t tiles_hidden = (blocks_hidden + tile_n_blocks - 1) / tile_n_blocks; - const size_t num_threads_p1 = std::min(num_workers, tiles_d_ffn); - const size_t num_threads_p2 = std::min(num_workers, tiles_hidden); - - for (size_t t = 0; t < M; ++t) { - const int8_t* x_int8 = dense_x_int8.data() + t * hidden_dim; - const float x_scale = dense_x_scales[t]; - __fp16* y = output + t * hidden_dim; - __fp16* h_full = dense_h_fp16.data(); - int8_t* h_int8 = dense_h_int8.data(); - - std::atomic p1_next{0}; - std::atomic p2_next{0}; - - auto run = [&](size_t nt, auto&& task) { - if (nt <= 1) { task(0); return; } - pool.enqueue_n_threads(nt - 1, nt - 1, [&](size_t s, size_t e) { - for (size_t w = s; w < e; ++w) task(w + 1); - }); - task(0); - pool.wait_all(); - }; - - run(num_threads_p1, [&](size_t worker_id) { - auto& sc = dense_fused_scratch[worker_id]; - __fp16* gate_tile_buf = sc.gate_tile.data(); - __fp16* up_tile_buf = sc.up_tile.data(); - while (true) { - size_t tile_id = p1_next.fetch_add(1, std::memory_order_relaxed); - if (tile_id >= tiles_d_ffn) break; - const size_t blk_start = tile_id * tile_n_blocks; - const size_t blk_end = std::min(blk_start + tile_n_blocks, blocks_d_ffn); - const size_t k_lo = blk_start * 4; - const size_t k_count = (blk_end - blk_start) * 4; - - cactus_gemv_int4_block_range( - x_int8, x_scale, gate_w, gate_s, gate_tile_buf - k_lo, - hidden_dim, d_ffn, group_size, blk_start, blk_end); - cactus_gelu_f16(gate_tile_buf, gate_tile_buf, k_count); - cactus_gemv_int4_block_range( - x_int8, x_scale, up_w, up_s, up_tile_buf - k_lo, - hidden_dim, d_ffn, group_size, blk_start, blk_end); - cactus_multiply_f16(gate_tile_buf, up_tile_buf, h_full + k_lo, k_count); - } - }); - - const float h_maxabs = cactus_fp16_max_abs(h_full, d_ffn); - const float h_scale = std::max(h_maxabs / 127.0f, 1e-10f); - fp16_to_int8_inplace(h_full, h_int8, d_ffn, h_scale); - - run(num_threads_p2, [&](size_t /*worker_id*/) { - while (true) { - size_t tile_id = p2_next.fetch_add(1, std::memory_order_relaxed); - if (tile_id >= tiles_hidden) break; - const size_t blk_start = tile_id * tile_n_blocks; - const size_t blk_end = std::min(blk_start + tile_n_blocks, blocks_hidden); - cactus_gemv_int4_block_range( - h_int8, h_scale, down_w, down_s, y, - d_ffn, hidden_dim, group_size, blk_start, blk_end); - } - }); - } -} - -void compute_rms_norm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); - - if (input_buffer.shape.size() != 2) { - throw std::runtime_error("RMS normalization requires 2D input tensor [batch_size, dims], got " + - std::to_string(input_buffer.shape.size()) + "D tensor"); - } - - size_t batch_size = input_buffer.shape[0]; - size_t dims = input_buffer.shape[1]; - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("RMS normalization only supports FP16 precision"); - } - - cactus_rms_norm_f16(input_buffer.data_as<__fp16>(), weight_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), batch_size, dims, node.params.epsilon); -} - -void compute_rope_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU RoPE operation not yet implemented"); - } - - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (shape.size() < 4) { - throw std::runtime_error("RoPE operation requires 4D tensor with shape [batch, seq_len, num_heads, head_dim], got " + - std::to_string(shape.size()) + "D tensor"); - } - - if (input_buffer.precision != Precision::FP16 || node.output_buffer.precision != Precision::FP16) { - throw std::runtime_error("RoPE operation only supports FP16 precision"); - } - - size_t batch_size = shape[0]; - size_t seq_len = shape[1]; - size_t num_heads = shape[2]; - size_t head_dim = shape[3]; - - cactus_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, num_heads, head_dim, node.params.position_offset, node.params.theta); -} - -void compute_softmax_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - if (shape.size() < 2) { - throw std::runtime_error("Softmax operation requires at least 2D tensor, got " + - std::to_string(shape.size()) + "D tensor"); - } - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("Softmax operation only supports FP16 precision"); - } - - size_t batch_size = 1; - for (size_t i = 0; i < shape.size() - 1; i++) { - batch_size *= shape[i]; - } - size_t vocab_size = shape[shape.size() - 1]; - - cactus_softmax_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, 1, vocab_size); -} - -void compute_rel_pos_bias_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 2) { - throw std::runtime_error("REL_POS_BIAS requires 2 inputs (query, relative_key)"); - } - - const auto& q_buffer = get_input(node, 0, nodes, node_index_map); - const auto& r_buffer = get_input(node, 1, nodes, node_index_map); - auto& y_buffer = node.output_buffer; - - if (q_buffer.shape.size() != 4) { - throw std::runtime_error("REL_POS_BIAS query must be [B, T, H, D]"); - } - if (r_buffer.shape.size() != 4) { - throw std::runtime_error("REL_POS_BIAS relative_key must be [B, R, H, D]"); - } - if (q_buffer.precision != Precision::FP16 || r_buffer.precision != Precision::FP16) { - throw std::runtime_error("REL_POS_BIAS currently only supports FP16 tensors"); - } - - const size_t B = q_buffer.shape[0]; - const size_t T = q_buffer.shape[1]; - const size_t H = q_buffer.shape[2]; - const size_t D = q_buffer.shape[3]; - const size_t Rb = r_buffer.shape[0]; - const size_t R = r_buffer.shape[1]; - - if (Rb != 1 && Rb != B) { - throw std::runtime_error("REL_POS_BIAS relative_key batch must be 1 or match query batch"); - } - if (r_buffer.shape[2] != H || r_buffer.shape[3] != D) { - throw std::runtime_error("REL_POS_BIAS expects matching [H, D] between query and relative_key"); - } - if (R < (2 * T - 1)) { - throw std::runtime_error("REL_POS_BIAS requires relative_key length >= 2*T-1"); - } - - const __fp16* q = q_buffer.data_as<__fp16>(); - const __fp16* r = r_buffer.data_as<__fp16>(); - __fp16* y = y_buffer.data_as<__fp16>(); - - const float scale = node.params.scale; - - const size_t q_batch_stride = T * H * D; - const size_t r_batch_stride = R * H * D; - const size_t y_batch_stride = H * T * T; - const size_t q_head_stride = D; - const size_t r_head_stride = D; - const size_t q_time_stride = H * D; - const size_t r_time_stride = H * D; - - CactusThreading::parallel_for(B * H * T, CactusThreading::Thresholds::ATTENTION, - [&](size_t start_idx, size_t end_idx) { - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t b = work_idx / (H * T); - const size_t rem = work_idx % (H * T); - const size_t h = rem / T; - const size_t t = rem % T; - - const size_t rb = (Rb == 1) ? 0 : b; - const __fp16* q_vec = q + b * q_batch_stride + t * q_time_stride + h * q_head_stride; - const __fp16* r_base = r + rb * r_batch_stride + h * r_head_stride; - __fp16* y_row = y + b * y_batch_stride + h * (T * T) + t * T; - - for (size_t j = 0; j < T; ++j) { - const size_t rel_idx = (T - 1) - t + j; - const __fp16* r_vec = r_base + rel_idx * r_time_stride; - - float acc = 0.0f; - for (size_t d = 0; d < D; ++d) { - acc += static_cast(q_vec[d]) * static_cast(r_vec[d]); - } - y_row[j] = static_cast<__fp16>(acc * scale); - } - } - }); -} - -void compute_attention_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU attention operation not yet implemented"); - } - - if (node.input_ids.size() < 3 || node.input_ids.size() > 4) { - throw std::runtime_error("Attention operation requires 3 or 4 inputs (query, key, value[, mask]), got " + - std::to_string(node.input_ids.size()) + " inputs"); - } - - const auto& query_buffer = get_input(node, 0, nodes, node_index_map); - const auto& key_buffer = get_input(node, 1, nodes, node_index_map); - const auto& value_buffer = get_input(node, 2, nodes, node_index_map); - const BufferDesc* mask_buffer = nullptr; - if (node.input_ids.size() == 4) { - mask_buffer = &get_input(node, 3, nodes, node_index_map); - } - const auto& q_shape = query_buffer.shape; - const auto& k_shape = key_buffer.shape; - - if (q_shape.size() < 4) { - throw std::runtime_error("Attention operation requires 4D tensors [batch, seq_len, num_heads, head_dim], got " + - std::to_string(q_shape.size()) + "D tensor"); - } - - if (query_buffer.precision != Precision::FP16) { - throw std::runtime_error("Attention operation only supports FP16 precision"); - } - - size_t batch_size = q_shape[0]; - size_t seq_len = q_shape[1]; - size_t num_q_heads = q_shape[2]; - size_t head_dim = q_shape[3]; - size_t num_kv_heads = k_shape[2]; - size_t kv_seq_len = key_buffer.shape[1]; - size_t v_head_dim = value_buffer.shape[3]; - bool mask_per_head = false; - const __fp16* mask_ptr = nullptr; - - if (mask_buffer) { - if (mask_buffer->precision != Precision::FP16) { - throw std::runtime_error("Attention mask tensor must be FP16"); - } - - if (mask_buffer->shape.size() == 3) { - if (mask_buffer->shape[0] != batch_size || - mask_buffer->shape[1] != seq_len || - mask_buffer->shape[2] != kv_seq_len) { - throw std::runtime_error("Attention mask [B, T, S] shape mismatch"); - } - mask_per_head = false; - } else if (mask_buffer->shape.size() == 4) { - if (mask_buffer->shape[0] != batch_size || - mask_buffer->shape[1] != num_q_heads || - mask_buffer->shape[2] != seq_len || - mask_buffer->shape[3] != kv_seq_len) { - throw std::runtime_error("Attention mask [B, H, T, S] shape mismatch"); - } - mask_per_head = true; - } else { - throw std::runtime_error("Attention mask must be rank 3 or 4"); - } - - mask_ptr = mask_buffer->data_as<__fp16>(); - } - - cactus_attention_f16(query_buffer.data_as<__fp16>(), key_buffer.data_as<__fp16>(), - value_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, kv_seq_len, num_q_heads, num_kv_heads, head_dim, node.params.scale, mask_ptr, - node.params.position_offset, node.params.window_size, node.params.is_causal, - node.params.attention_mask_is_additive, mask_per_head, v_head_dim, node.params.logit_cap); -} - -void compute_attention_int8_hybrid_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& query_buffer = get_input(node, 0, nodes, node_index_map); - const auto& key_new_buffer = get_input(node, 1, nodes, node_index_map); - const auto& value_new_buffer = get_input(node, 2, nodes, node_index_map); - const auto& q_shape = query_buffer.shape; - - if (q_shape.size() < 4) { - throw std::runtime_error("ATTENTION_INT8_HYBRID requires 4D query tensor"); - } - - size_t batch_size = q_shape[0]; - size_t seq_len = q_shape[1]; - size_t num_q_heads = q_shape[2]; - size_t head_dim = node.params.head_dim; - size_t v_head_dim = node.params.v_head_dim; - size_t num_kv_heads = node.params.num_kv_heads; - size_t cache_len = node.params.cache_seq_len; - size_t new_len = key_new_buffer.shape[1]; - - cactus_attention_hybrid_int8_fp16( - query_buffer.data_as<__fp16>(), - node.params.cached_keys_int8, - node.params.cached_values_int8, - node.params.cached_k_scales, - node.params.cached_v_scales, - key_new_buffer.data_as<__fp16>(), - value_new_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, cache_len, new_len, - num_q_heads, num_kv_heads, head_dim, - node.params.scale, node.params.position_offset, true, - node.params.window_size, KV_QUANT_GROUP_SIZE, v_head_dim - ); -} - -void compute_layernorm_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& weight_buffer = get_input(node, 1, nodes, node_index_map); - bool has_bias = node.input_ids.size() > 2; - float epsilon = node.params.epsilon; - - if (input_buffer.shape.empty()) { - throw std::runtime_error("LayerNorm requires non-empty input tensor"); - } - - size_t feature_size = input_buffer.shape.back(); - size_t batch_size = input_buffer.total_size / feature_size; - - if (weight_buffer.total_size != feature_size) { - throw std::runtime_error("LayerNorm weight size mismatch with input feature dimension"); - } - - using BufferDesc = std::remove_reference_t; - const BufferDesc* bias_buffer_ptr = nullptr; - if (has_bias) { - const auto& bias_buffer = get_input(node, 2, nodes, node_index_map); - if (bias_buffer.total_size != feature_size) { - throw std::runtime_error("LayerNorm bias size mismatch with input feature dimension"); - } - bias_buffer_ptr = &bias_buffer; - } - - if (input_buffer.precision == Precision::FP16 && - weight_buffer.precision == Precision::FP16 && - node.output_buffer.precision == Precision::FP16 && - (!has_bias || bias_buffer_ptr->precision == Precision::FP16)) { - cactus_layer_norm_f16( - input_buffer.data_as<__fp16>(), - weight_buffer.data_as<__fp16>(), - has_bias ? bias_buffer_ptr->data_as<__fp16>() : nullptr, - node.output_buffer.data_as<__fp16>(), - batch_size, - feature_size, - epsilon); - return; - } - - std::vector input_float(input_buffer.total_size); - std::vector weight_float(feature_size); - std::vector bias_float(feature_size, 0.0f); - - if (input_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 input"); - } else if (input_buffer.precision == Precision::FP16) { - const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - input_float[i] = static_cast(input_fp16[i]); - } - } else { - std::memcpy(input_float.data(), input_buffer.data_as(), input_buffer.total_size * sizeof(float)); - } - - if (weight_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 weight"); - } else if (weight_buffer.precision == Precision::FP16) { - const __fp16* weight_fp16 = weight_buffer.data_as<__fp16>(); - for (size_t i = 0; i < feature_size; ++i) { - weight_float[i] = static_cast(weight_fp16[i]); - } - } else { - std::memcpy(weight_float.data(), weight_buffer.data_as(), feature_size * sizeof(float)); - } - - if (has_bias) { - const auto& bias_buffer = *bias_buffer_ptr; - if (bias_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 bias"); - } else if (bias_buffer.precision == Precision::FP16) { - const __fp16* bias_fp16 = bias_buffer.data_as<__fp16>(); - for (size_t i = 0; i < feature_size; ++i) { - bias_float[i] = static_cast(bias_fp16[i]); - } - } else { - std::memcpy(bias_float.data(), bias_buffer.data_as(), feature_size * sizeof(float)); - } - } - - std::vector output_float(input_buffer.total_size); - for (size_t b = 0; b < batch_size; ++b) { - const float* input_row = input_float.data() + b * feature_size; - float* output_row = output_float.data() + b * feature_size; - - float mean = 0.0f; - for (size_t i = 0; i < feature_size; ++i) { - mean += input_row[i]; - } - mean /= feature_size; - - float variance = 0.0f; - for (size_t i = 0; i < feature_size; ++i) { - float diff = input_row[i] - mean; - variance += diff * diff; - } - variance /= feature_size; - - float std_inv = 1.0f / std::sqrt(variance + epsilon); - for (size_t i = 0; i < feature_size; ++i) { - output_row[i] = (input_row[i] - mean) * std_inv * weight_float[i] + bias_float[i]; - } - } - - if (node.output_buffer.precision == Precision::INT8) { - throw std::runtime_error("LayerNorm currently does not support INT8 output"); - } else if (node.output_buffer.precision == Precision::FP16) { - __fp16* output_fp16 = node.output_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - output_fp16[i] = static_cast<__fp16>(output_float[i]); - } - } else { - std::memcpy(node.output_buffer.data_as(), output_float.data(), input_buffer.total_size * sizeof(float)); - } -} - -void compute_conv1d_causal_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU causal convolution operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("Causal conv requires 3D input [batch, seq_len, in_channels]"); - } - if (W.shape.size() != 3) { - throw std::runtime_error("Weight must be 3D"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C_in = X.shape[2]; - const size_t W0 = W.shape[0]; - const size_t W1 = W.shape[1]; - const size_t W2 = W.shape[2]; - const size_t dil = node.params.dilation; - if (dil < 1) throw std::runtime_error("dilation must be >= 1"); - - size_t M = 1; - size_t C_out = 0; - const bool standard_layout = (W1 == 1); - const bool transposed_layout = (W2 == 1); - if ((!standard_layout && !transposed_layout) || (W0 % C_in != 0)) { - throw std::runtime_error("Only depthwise causal convolution is supported currently"); - } - const size_t K = standard_layout ? W2 : W1; - M = W0 / C_in; - C_out = C_in * M; - - Y.shape = { N, L, C_out }; - Y.precision = X.precision; - - auto transpose_depthwise_weights_fp16 = [&](const __fp16* src) { - std::vector<__fp16> transposed(W0 * K); - for (size_t oc = 0; oc < W0; ++oc) { - for (size_t k = 0; k < K; ++k) { - transposed[oc * K + k] = src[(oc * W1 + k) * W2]; - } - } - return transposed; - }; - - if (W.precision == Precision::INT8) { - const size_t W_size = W0 * W1 * W2; - const int8_t* W_int8 = W.data_as(); - - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = W1 * K; - const size_t group_size = W.group_size; - const size_t num_groups = K_total / group_size; - - for (size_t row = 0; row < W0; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - if (transposed_layout && !standard_layout) { - auto fixed = transpose_depthwise_weights_fp16(W_fp16.data()); - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } else { - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), W_fp16.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } - } else if (W.precision == Precision::FP16) { - if (transposed_layout && !standard_layout) { - auto fixed = transpose_depthwise_weights_fp16(W.data_as<__fp16>()); - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), fixed.data(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } else { - cactus_conv1d_causal_depthwise_f16( - X.data_as<__fp16>(), W.data_as<__fp16>(), Y.data_as<__fp16>(), - N, L, C_in, K, dil); - } - } else { - throw std::runtime_error("Depthwise causal conv supports INT8/FP16 weights"); - } -} - -void compute_conv1d_k3_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU causal convolution operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) - throw std::runtime_error("Conv requires 3D input [N, C_in, L]!"); - - if (W.shape.size() != 3) - throw std::runtime_error("Weight must be [C_out, C_in, 3]!"); - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - - if (K != 3) - throw std::runtime_error("Conv1d_k3 only supports K=3!"); - - size_t L_out = ((L - 1) / stride) + 1; - Y.shape = { N, C_out, L_out }; - Y.precision = X.precision; - - if (X.precision != Precision::FP16) { - throw std::runtime_error("Conv1d_k3 only supports FP16 activations"); - } - - if (W.precision == Precision::INT8) { - const size_t W_size = C_out * C_in * K; - const int8_t* W_int8 = W.data_as(); - - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = C_in * K; - const size_t group_size = W.group_size; - const size_t num_groups = K_total / group_size; - - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_f16_k3( - X.data_as<__fp16>(), - W_fp16.data(), - Y.data_as<__fp16>(), - N, L, C_in, C_out, stride - ); - } else if (W.precision == Precision::FP16) { - cactus_conv1d_f16_k3( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - Y.data_as<__fp16>(), - N, L, C_in, C_out, stride - ); - } else { - throw std::runtime_error("Conv1d_k3 only supports FP16 and INT8 weights"); - } -} - -void compute_conv1d_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d expects input [N, C_in, L]"); - } - if (W.shape.size() != 3) { - throw std::runtime_error("conv1d weight must be [C_out, C_in, K]"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - - if (W.shape[1] != C_in) { - throw std::runtime_error("conv1d weight C_in mismatch"); - } - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("Conv1d only supports FP16"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d bias only supports FP16/FP32"); - } - } - - cactus_conv1d_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, - Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride); -} - -void compute_conv1d_same_depthwise_k9_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv1d_same_depthwise_k9 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d_same_depthwise_k9 expects input [N, L, C]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C = X.shape[2]; - const size_t K = 9; - - if (W.shape.size() == 2) { - if (W.shape[0] != C || W.shape[1] != K) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 9]"); - } - } else if (W.shape.size() == 3) { - if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != K) { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be [C, 1, 9]"); - } - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 weight must be rank 2 or 3"); - } - - Y.shape = {N, L, C}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C) { - throw std::runtime_error("conv1d_same_depthwise_k9 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_same_depthwise_k9 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv1d_same_depthwise_f16_k9( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t W_size = C * K; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t K_total = K; - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv1d_same_depthwise_k9 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_same_depthwise_f16_k9( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C - ); - return; - } - - throw std::runtime_error("conv1d_same_depthwise_k9 only supports FP16/INT8 weights"); -} - -void compute_conv2d_k3s2p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_k3s2p1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_k3s2p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - const size_t C_out = Y.shape[1]; - - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_k3s2p1 input spatial dimensions must be > 0"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_k3s2p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - if (W.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s2p1 FP16 weight must be [C_out, C_in, 3, 3]"); - } - cactus_conv2d_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in * 9; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_k3s2p1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - throw std::runtime_error("conv2d_k3s2p1 only supports FP16/INT8 weights"); -} - -void compute_conv2d_depthwise_k3s2p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_depthwise_k3s2p1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 expects input [N, C, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 input spatial dimensions must be > 0"); - } - - if (W.shape.size() == 3) { - if (W.shape[0] != C || W.shape[1] != 3 || W.shape[2] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 3, 3]"); - } - } else if (W.shape.size() == 4) { - if (W.shape[0] != C || W.shape[1] != 1 || W.shape[2] != 3 || W.shape[3] != 3) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be [C, 1, 3, 3]"); - } - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 weight must be rank 3 or 4"); - } - - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W_in - 1) / 2 + 1; - Y.shape = {N, C, H_out, W_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C) { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_depthwise_k3s2p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv2d_depthwise_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C, H, W_in - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = 9; - const size_t W_size = C * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_depthwise_k3s2p1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_depthwise_f16_k3s2p1_nchw( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C, H, W_in - ); - return; - } - - throw std::runtime_error("conv2d_depthwise_k3s2p1 only supports FP16/INT8 weights"); -} - -void compute_conv2d_pointwise_1x1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv2d_pointwise_1x1 operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_pointwise_1x1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_pointwise_1x1 input spatial dimensions must be > 0"); - } - - size_t C_out = 0; - if (W.shape.size() == 2) { - C_out = W.shape[0]; - if (W.shape[1] != C_in) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in]"); - } - } else if (W.shape.size() == 4) { - C_out = W.shape[0]; - if (W.shape[1] != C_in || W.shape[2] != 1 || W.shape[3] != 1) { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be [C_out, C_in, 1, 1]"); - } - } else { - throw std::runtime_error("conv2d_pointwise_1x1 weight must be rank 2 or 4"); - } - - Y.shape = {N, C_out, H, W_in}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv2d_pointwise_1x1 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_pointwise_1x1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv2d_pointwise_1x1 requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out - ); - return; - } - - throw std::runtime_error("conv2d_pointwise_1x1 only supports FP16/INT8 weights"); -} - -void compute_conv1d_pointwise_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU conv1d_pointwise operation not yet implemented"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 3) { - throw std::runtime_error("conv1d_pointwise expects input [N, L, C_in]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv1d_pointwise only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t L = X.shape[1]; - const size_t C_in = X.shape[2]; - - size_t C_out = 0; - if (W.shape.size() == 2) { - C_out = W.shape[0]; - if (W.shape[1] != C_in) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in]"); - } - } else if (W.shape.size() == 3) { - C_out = W.shape[0]; - if (W.shape[1] != C_in || W.shape[2] != 1) { - throw std::runtime_error("conv1d_pointwise weight must be [C_out, C_in, 1]"); - } - } else { - throw std::runtime_error("conv1d_pointwise weight must be rank 2 or 3"); - } - - Y.shape = {N, L, C_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d_pointwise bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_pointwise bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - cactus_conv1d_pointwise_f16_gemm( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); - return; - } - - if (W.precision == Precision::INT8) { - const size_t K_total = C_in; - const size_t W_size = C_out * K_total; - const int8_t* W_int8 = W.data_as(); - std::vector<__fp16> W_fp16(W_size); - - if (W.is_grouped_int8()) { - const __fp16* scales = W.scales_as_fp16(); - const size_t group_size = W.group_size; - if (group_size == 0 || (K_total % group_size) != 0 || scales == nullptr) { - throw std::runtime_error("Grouped INT8 conv1d_pointwise requires valid per-group scales"); - } - - const size_t num_groups = K_total / group_size; - for (size_t row = 0; row < C_out; ++row) { - for (size_t col = 0; col < K_total; ++col) { - size_t idx = row * K_total + col; - size_t group_idx = col / group_size; - float scale = static_cast(scales[row * num_groups + group_idx]); - W_fp16[idx] = static_cast<__fp16>(W_int8[idx] * scale); - } - } - } else { - for (size_t i = 0; i < W_size; ++i) { - W_fp16[i] = static_cast<__fp16>(W_int8[i]); - } - } - - cactus_conv1d_pointwise_f16_gemm( - X.data_as<__fp16>(), - W_fp16.data(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); - return; - } - - throw std::runtime_error("conv1d_pointwise only supports FP16/INT8 weights"); -} - -void compute_glu_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.empty()) { - throw std::runtime_error("GLU expects non-scalar input"); - } - - int axis = node.params.axis; - if (axis < 0) axis += static_cast(X.shape.size()); - if (axis < 0 || static_cast(axis) >= X.shape.size()) { - throw std::runtime_error("GLU axis out of range"); - } - - const size_t axis_size = X.shape[static_cast(axis)]; - if ((axis_size % 2) != 0) { - throw std::runtime_error("GLU split dimension must be even"); - } - const size_t split = axis_size / 2; - - size_t outer = 1; - for (int i = 0; i < axis; ++i) { - outer *= X.shape[static_cast(i)]; - } - size_t inner = 1; - for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { - inner *= X.shape[i]; - } - - std::vector out_shape = X.shape; - out_shape[static_cast(axis)] = split; - Y.shape = out_shape; - Y.precision = X.precision; - - if (X.precision == Precision::FP16) { - cactus_glu_f16(X.data_as<__fp16>(), Y.data_as<__fp16>(), outer, split, inner); - return; - } - - if (X.precision == Precision::FP32) { - cactus_glu_f32(X.data_as(), Y.data_as(), outer, split, inner); - return; - } - - throw std::runtime_error("GLU only supports FP16/FP32"); -} - -void compute_batchnorm_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 5) { - throw std::runtime_error("BatchNorm expects 5 inputs: input, weight, bias, running_mean, running_var"); - } - - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const auto& B = get_input(node, 2, nodes, node_index_map); - const auto& RM = get_input(node, 3, nodes, node_index_map); - const auto& RV = get_input(node, 4, nodes, node_index_map); - auto& Y = node.output_buffer; - - if (X.shape.empty()) { - throw std::runtime_error("BatchNorm expects non-scalar input"); - } - - int axis = node.params.axis; - if (axis < 0) axis += static_cast(X.shape.size()); - if (axis < 0 || static_cast(axis) >= X.shape.size()) { - throw std::runtime_error("BatchNorm axis out of range"); - } - - const size_t C = X.shape[static_cast(axis)]; - if (W.total_size != C || B.total_size != C || RM.total_size != C || RV.total_size != C) { - throw std::runtime_error("BatchNorm parameter size mismatch"); - } - - auto load_1d_float = [C](const BufferDesc& buf, const char* name) -> std::vector { - if (buf.total_size != C) { - throw std::runtime_error(std::string("BatchNorm parameter size mismatch for ") + name); - } - std::vector out(C); - if (buf.precision == Precision::FP16) { - const __fp16* p = buf.data_as<__fp16>(); - for (size_t i = 0; i < C; ++i) out[i] = static_cast(p[i]); - } else if (buf.precision == Precision::FP32) { - std::memcpy(out.data(), buf.data_as(), C * sizeof(float)); - } else { - throw std::runtime_error(std::string("BatchNorm parameter ") + name + " must be FP16 or FP32"); - } - return out; - }; - - const std::vector gamma = load_1d_float(W, "weight"); - const std::vector beta = load_1d_float(B, "bias"); - const std::vector mean = load_1d_float(RM, "running_mean"); - const std::vector var = load_1d_float(RV, "running_var"); - - size_t outer = 1; - for (int i = 0; i < axis; ++i) { - outer *= X.shape[static_cast(i)]; - } - size_t inner = 1; - for (size_t i = static_cast(axis) + 1; i < X.shape.size(); ++i) { - inner *= X.shape[i]; - } - - Y.shape = X.shape; - Y.precision = X.precision; - - if (X.precision == Precision::FP16) { - cactus_batchnorm_f16( - X.data_as<__fp16>(), - gamma.data(), - beta.data(), - mean.data(), - var.data(), - Y.data_as<__fp16>(), - outer, - C, - inner, - node.params.epsilon - ); - return; - } - - if (X.precision == Precision::FP32) { - cactus_batchnorm_f32( - X.data_as(), - gamma.data(), - beta.data(), - mean.data(), - var.data(), - Y.data_as(), - outer, - C, - inner, - node.params.epsilon - ); - return; - } - - throw std::runtime_error("BatchNorm only supports FP16/FP32 activations"); -} - -void compute_stft_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - auto& Y = node.output_buffer; - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - const size_t C_out = W.shape[0]; - const size_t K = W.shape[2]; - const size_t stride = node.params.stride; - const size_t num_fft_bins = node.params.num_fft_bins; - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("stft only supports FP16"); - } - - cactus_stft_f16(X.data_as<__fp16>(), W.data_as<__fp16>(), - Y.data_as<__fp16>(), N, L, C_in, C_out, K, stride, num_fft_bins); -} - -void compute_conv1d_k7s3_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - - auto& Y = node.output_buffer; - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t L = X.shape[2]; - - if (W.shape.size() != 3) throw std::runtime_error("Weight must be 3D"); - const size_t C_in_W = W.shape[0]; - const size_t K = W.shape[1]; - const size_t C_out = W.shape[2]; - const size_t stride = node.params.stride; - - if (C_in != C_in_W) throw std::runtime_error("Channel mismatch in conv1d_k7s3"); - if (K != 7 || stride != 3) throw std::runtime_error("conv1d_k7s3 requires K=7, stride=3"); - - if (X.precision != Precision::FP16 || W.precision != Precision::FP16) { - throw std::runtime_error("Conv1d specialized only supports FP16"); - } - - size_t L_out = (L < 7) ? 0 : (L - 7) / 3 + 1; - Y.shape = {N, C_out, L_out}; - Y.precision = Precision::FP16; - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->total_size != C_out) { - throw std::runtime_error("conv1d_k7s3 bias size mismatch"); - } - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv1d_k7s3 bias only supports FP16/FP32"); - } - } - - cactus_conv1d_f16_k7s3_oc8( - X.data_as<__fp16>(), - W.data_as<__fp16>(), - bias_ptr, - Y.data_as<__fp16>(), - N, L, C_in, C_out - ); -} - -void compute_gated_deltanet_decode_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 6) { - throw std::runtime_error("GATED_DELTANET_DECODE expects 6 inputs"); - } - - const auto& q = get_input(node, 0, nodes, node_index_map); - const auto& k = get_input(node, 1, nodes, node_index_map); - const auto& v = get_input(node, 2, nodes, node_index_map); - const auto& g = get_input(node, 3, nodes, node_index_map); - const auto& b = get_input(node, 4, nodes, node_index_map); - const auto& s = get_input(node, 5, nodes, node_index_map); - - validate_gated_deltanet_inputs(q, k, v, g, b, s); - if (q.shape[1] != 1) { - throw std::runtime_error("GATED_DELTANET_DECODE expects T=1"); - } - - const size_t B = q.shape[0]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - const size_t qk_heads_from_params = node.params.num_kv_heads; - if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { - throw std::runtime_error("GATED_DELTANET_DECODE num_qk_heads param mismatch"); - } - - std::vector<__fp16> q_cast; - std::vector<__fp16> k_cast; - std::vector<__fp16> v_cast; - std::vector<__fp16> g_cast; - std::vector<__fp16> b_cast; - std::vector<__fp16> s_cast; - const __fp16* q_data = as_fp16_ptr(q, q_cast); - const __fp16* k_data = as_fp16_ptr(k, k_cast); - const __fp16* v_data = as_fp16_ptr(v, v_cast); - const __fp16* g_data = as_fp16_ptr(g, g_cast); - const __fp16* b_data = as_fp16_ptr(b, b_cast); - const __fp16* s_data = as_fp16_ptr(s, s_cast); - __fp16* out = node.output_buffer.data_as<__fp16>(); - - cactus_gated_deltanet_decode_f16( - q_data, k_data, v_data, g_data, b_data, s_data, out, - B, Hq, Hv, K, V, node.params.scale); -} - -void compute_gated_deltanet_prefill_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - if (node.input_ids.size() != 6) { - throw std::runtime_error("GATED_DELTANET_PREFILL expects 6 inputs"); - } - - const auto& q = get_input(node, 0, nodes, node_index_map); - const auto& k = get_input(node, 1, nodes, node_index_map); - const auto& v = get_input(node, 2, nodes, node_index_map); - const auto& g = get_input(node, 3, nodes, node_index_map); - const auto& b = get_input(node, 4, nodes, node_index_map); - const auto& s = get_input(node, 5, nodes, node_index_map); - - validate_gated_deltanet_inputs(q, k, v, g, b, s); - - const size_t B = q.shape[0]; - const size_t T = q.shape[1]; - const size_t Hq = q.shape[2]; - const size_t K = q.shape[3]; - const size_t Hv = v.shape[2]; - const size_t V = v.shape[3]; - const size_t qk_heads_from_params = node.params.num_kv_heads; - if (qk_heads_from_params != 0 && qk_heads_from_params != Hq) { - throw std::runtime_error("GATED_DELTANET_PREFILL num_qk_heads param mismatch"); - } - - std::vector<__fp16> q_cast; - std::vector<__fp16> k_cast; - std::vector<__fp16> v_cast; - std::vector<__fp16> g_cast; - std::vector<__fp16> b_cast; - std::vector<__fp16> s_cast; - const __fp16* q_data = as_fp16_ptr(q, q_cast); - const __fp16* k_data = as_fp16_ptr(k, k_cast); - const __fp16* v_data = as_fp16_ptr(v, v_cast); - const __fp16* g_data = as_fp16_ptr(g, g_cast); - const __fp16* b_data = as_fp16_ptr(b, b_cast); - const __fp16* s_data = as_fp16_ptr(s, s_cast); - __fp16* out = node.output_buffer.data_as<__fp16>(); - - cactus_gated_deltanet_prefill_f16( - q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, node.params.chunk_size, node.params.scale); -} - -void compute_rope_gptj_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& shape = input_buffer.shape; - - size_t batch_size = shape[0]; - size_t seq_len = shape[1]; - size_t num_heads = shape[2]; - size_t head_dim = shape[3]; - size_t rot_dim = static_cast(node.params.scalar); - - cactus_gpt_j_rope_f16(input_buffer.data_as<__fp16>(), node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, num_heads, head_dim, rot_dim, - node.params.position_offset, node.params.theta); -} - -void compute_groupnorm_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& weight = get_input(node, 1, nodes, node_index_map); - const auto& bias = get_input(node, 2, nodes, node_index_map); - float epsilon = node.params.epsilon; - - size_t batch_size = input.shape[0]; - size_t channels = input.shape[1]; - size_t spatial_size = 1; - for (size_t i = 2; i < input.shape.size(); ++i) spatial_size *= input.shape[i]; - - size_t num_groups = node.params.num_groups; - if (num_groups == 0) num_groups = 32; - - if (channels % num_groups != 0) { - throw std::runtime_error("GroupNorm: channels must be divisible by num_groups"); - } - - size_t channels_per_group = channels / num_groups; - - const __fp16* src = input.data_as<__fp16>(); - const __fp16* w = weight.data_as<__fp16>(); - const __fp16* b = bias.data_as<__fp16>(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - for (size_t n = 0; n < batch_size; ++n) { - for (size_t g = 0; g < num_groups; ++g) { - float sum = 0.0f, sum_sq = 0.0f; - size_t count = 0; - - for (size_t c = 0; c < channels_per_group; ++c) { - size_t ch = g * channels_per_group + c; - for (size_t s = 0; s < spatial_size; ++s) { - size_t idx = n * channels * spatial_size + ch * spatial_size + s; - float val = static_cast(src[idx]); - sum += val; - sum_sq += val * val; - count++; - } - } - - float mean = sum / count; - float var = (sum_sq / count) - (mean * mean); - float inv_std = 1.0f / std::sqrt(var + epsilon); - - for (size_t c = 0; c < channels_per_group; ++c) { - size_t ch = g * channels_per_group + c; - float wt = static_cast(w[ch]); - float bi = static_cast(b[ch]); - - for (size_t s = 0; s < spatial_size; ++s) { - size_t idx = n * channels * spatial_size + ch * spatial_size + s; - float val = static_cast(src[idx]); - dst[idx] = static_cast<__fp16>((val - mean) * inv_std * wt + bi); - } - } - } - } -} - -void compute_lstm_cell_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& h_prev_buffer = get_input(node, 1, nodes, node_index_map); - const auto& c_prev_buffer = get_input(node, 2, nodes, node_index_map); - const auto& weight_ih_buffer = get_input(node, 3, nodes, node_index_map); - const auto& weight_hh_buffer = get_input(node, 4, nodes, node_index_map); - const auto& bias_ih_buffer = get_input(node, 5, nodes, node_index_map); - const auto& bias_hh_buffer = get_input(node, 6, nodes, node_index_map); - - if (input_buffer.precision != Precision::FP16 || h_prev_buffer.precision != Precision::FP16 || - c_prev_buffer.precision != Precision::FP16 || weight_ih_buffer.precision != Precision::FP16 || - weight_hh_buffer.precision != Precision::FP16 || bias_ih_buffer.precision != Precision::FP16 || - bias_hh_buffer.precision != Precision::FP16) { - throw std::runtime_error("LSTM cell requires all inputs to be FP16"); - } - - if (input_buffer.shape.size() != 2 || h_prev_buffer.shape.size() != 2 || c_prev_buffer.shape.size() != 2) { - throw std::runtime_error("LSTM cell input/state shapes must be 2D [batch, features]"); - } - - const size_t batch_size = input_buffer.shape[0]; - const size_t input_size = input_buffer.shape[1]; - const size_t hidden_size = h_prev_buffer.shape[1]; - - const __fp16* x_input = input_buffer.data_as<__fp16>(); - const __fp16* h_prev = h_prev_buffer.data_as<__fp16>(); - const __fp16* c_prev = c_prev_buffer.data_as<__fp16>(); - const __fp16* weight_ih = weight_ih_buffer.data_as<__fp16>(); - const __fp16* weight_hh = weight_hh_buffer.data_as<__fp16>(); - const __fp16* bias_ih = bias_ih_buffer.data_as<__fp16>(); - const __fp16* bias_hh = bias_hh_buffer.data_as<__fp16>(); - - node.output_buffer.shape = {batch_size, hidden_size, 2}; - node.output_buffer.total_size = batch_size * hidden_size * 2; - node.output_buffer.precision = Precision::FP16; - node.output_buffer.allocate(); - - std::vector<__fp16> h_new_temp(batch_size * hidden_size); - std::vector<__fp16> c_new_temp(batch_size * hidden_size); - - cactus_lstm_cell_f16( - x_input, h_prev, c_prev, - weight_ih, weight_hh, - bias_ih, bias_hh, - h_new_temp.data(), c_new_temp.data(), - batch_size, input_size, hidden_size - ); - - __fp16* output = node.output_buffer.data_as<__fp16>(); - for (size_t b = 0; b < batch_size; ++b) { - for (size_t i = 0; i < hidden_size; ++i) { - const size_t idx = b * hidden_size + i; - output[b * hidden_size * 2 + i * 2] = h_new_temp[idx]; - output[b * hidden_size * 2 + i * 2 + 1] = c_new_temp[idx]; - } - } -} - -void compute_altup_predict_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - size_t n = node.params.num_altup_inputs; - const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); - - std::vector stream_ptrs(n); - for (size_t i = 0; i < n; i++) { - stream_ptrs[i] = get_input(node, 1 + i, nodes, node_index_map).data_as<__fp16>(); - } - - const auto& stream0_buf = get_input(node, 1, nodes, node_index_map); - size_t seq_len = stream0_buf.shape[0]; - size_t hidden_dim = stream0_buf.shape[1]; - - cactus_altup_predict_f16( - coefs_buf.data_as<__fp16>(), - stream_ptrs.data(), - node.output_buffer.data_as<__fp16>(), - n, seq_len, hidden_dim); -} - -void compute_gaussian_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buf = get_input(node, 0, nodes, node_index_map); - const __fp16* input = input_buf.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - size_t rows = input_buf.shape[0]; - size_t cols = input_buf.shape[1]; - float ppf = node.params.scalar; - - cactus_gaussian_topk_f16(input, output, rows, cols, ppf); -} - -void compute_altup_correct_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - size_t n = node.params.num_altup_inputs; - const auto& coefs_buf = get_input(node, 0, nodes, node_index_map); - const auto& innov_buf = get_input(node, 1, nodes, node_index_map); - - std::vector pred_ptrs(n); - for (size_t i = 0; i < n; i++) { - pred_ptrs[i] = get_input(node, 2 + i, nodes, node_index_map).data_as<__fp16>(); - } - - size_t seq_len = innov_buf.shape[0]; - size_t hidden_dim = innov_buf.shape[1]; - - cactus_altup_correct_f16( - coefs_buf.data_as<__fp16>(), - innov_buf.data_as<__fp16>(), - pred_ptrs.data(), - node.output_buffer.data_as<__fp16>(), - n, seq_len, hidden_dim); -} - -void compute_bilstm_sequence_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& w_ih_fwd = get_input(node, 1, nodes, node_index_map); - const auto& w_hh_fwd = get_input(node, 2, nodes, node_index_map); - const auto& b_ih_fwd = get_input(node, 3, nodes, node_index_map); - const auto& b_hh_fwd = get_input(node, 4, nodes, node_index_map); - const auto& w_ih_bwd = get_input(node, 5, nodes, node_index_map); - const auto& w_hh_bwd = get_input(node, 6, nodes, node_index_map); - const auto& b_ih_bwd = get_input(node, 7, nodes, node_index_map); - const auto& b_hh_bwd = get_input(node, 8, nodes, node_index_map); - - size_t batch_size = input.shape[0]; - size_t seq_len = input.shape[1]; - size_t input_size = input.shape[2]; - size_t hidden_size = w_ih_fwd.shape[0] / 4; - - cactus_bilstm_sequence_f16( - input.data_as<__fp16>(), - w_ih_fwd.data_as<__fp16>(), w_hh_fwd.data_as<__fp16>(), - b_ih_fwd.data_as<__fp16>(), b_hh_fwd.data_as<__fp16>(), - w_ih_bwd.data_as<__fp16>(), w_hh_bwd.data_as<__fp16>(), - b_ih_bwd.data_as<__fp16>(), b_hh_bwd.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, seq_len, input_size, hidden_size); -} - -void compute_maxpool1d_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - - size_t batch_size = input.shape[0]; - size_t channels = input.shape[1]; - size_t input_length = input.shape[2]; - size_t kernel_size = node.params.kernel_size; - size_t stride = node.params.stride; - - cactus_maxpool1d_f16( - input.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - batch_size, channels, input_length, - kernel_size, stride); -} - -void compute_conv2d_k3s1p1_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& X = get_input(node, 0, nodes, node_index_map); - const auto& W = get_input(node, 1, nodes, node_index_map); - const BufferDesc* B = nullptr; - if (node.input_ids.size() >= 3) { - B = &get_input(node, 2, nodes, node_index_map); - } - auto& Y = node.output_buffer; - - if (X.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 expects input [N, C_in, H, W]"); - } - if (X.precision != Precision::FP16) { - throw std::runtime_error("conv2d_k3s1p1 only supports FP16 activations"); - } - - const size_t N = X.shape[0]; - const size_t C_in = X.shape[1]; - const size_t H = X.shape[2]; - const size_t W_in = X.shape[3]; - const size_t C_out = Y.shape[1]; - - if (H == 0 || W_in == 0) { - throw std::runtime_error("conv2d_k3s1p1 input spatial dimensions must be > 0"); - } - - const __fp16* bias_ptr = nullptr; - std::vector<__fp16> bias_fp16; - if (B) { - if (B->precision == Precision::FP16) { - bias_ptr = B->data_as<__fp16>(); - } else if (B->precision == Precision::FP32) { - bias_fp16.resize(C_out); - cactus_fp32_to_fp16(B->data_as(), bias_fp16.data(), C_out); - bias_ptr = bias_fp16.data(); - } else { - throw std::runtime_error("conv2d_k3s1p1 bias only supports FP16/FP32"); - } - } - - if (W.precision == Precision::FP16) { - if (W.shape.size() != 4) { - throw std::runtime_error("conv2d_k3s1p1 FP16 weight must be [C_out, C_in, 3, 3]"); - } - cactus_conv2d_f16_k3s1p1_nchw( - X.data_as<__fp16>(), W.data_as<__fp16>(), bias_ptr, - Y.data_as<__fp16>(), - N, C_in, H, W_in, C_out); - return; - } - - throw std::runtime_error("conv2d_k3s1p1 only supports FP16 weights"); -} - -void compute_stats_pool_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const __fp16* src = input.data_as<__fp16>(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - size_t batch = input.shape[0]; - size_t total_per_batch = input.total_size / batch; - size_t T = input.shape.back(); - size_t features = total_per_batch / T; - - for (size_t b = 0; b < batch; ++b) { - const __fp16* batch_src = src + b * total_per_batch; - __fp16* batch_dst = dst + b * features * 2; - - for (size_t f = 0; f < features; ++f) { - float sum = 0.0f, sum_sq = 0.0f; - for (size_t t = 0; t < T; ++t) { - float v = static_cast(batch_src[f * T + t]); - sum += v; - sum_sq += v * v; - } - float mean = sum / static_cast(T); - float var = T > 1 ? (sum_sq - static_cast(T) * mean * mean) / static_cast(T - 1) : 0.0f; - float std_val = sqrtf(fmaxf(var, 0.0f)); - batch_dst[f] = static_cast<__fp16>(mean); - batch_dst[features + f] = static_cast<__fp16>(std_val); - } - } -} - -void compute_weighted_stats_pool_node(GraphNode& node, const std::vector>& nodes, - const std::unordered_map& node_index_map) { - const auto& input = get_input(node, 0, nodes, node_index_map); - const auto& weight_buf = get_input(node, 1, nodes, node_index_map); - const __fp16* src = input.data_as<__fp16>(); - const float* weights = weight_buf.data_as(); - __fp16* dst = node.output_buffer.data_as<__fp16>(); - - size_t batch = input.shape[0]; - size_t total_per_batch = input.total_size / batch; - size_t T = input.shape.back(); - size_t features = total_per_batch / T; - - constexpr float eps = 1e-8f; - - for (size_t b = 0; b < batch; ++b) { - const __fp16* batch_src = src + b * total_per_batch; - const float* batch_w = weights + b * T; - __fp16* batch_dst = dst + b * features * 2; - - float v1 = 0.0f, v2 = 0.0f; - for (size_t t = 0; t < T; ++t) { - float w = batch_w[t]; - v1 += w; - v2 += w * w; - } - float v1_safe = v1 + eps; - float var_denom = v1_safe - v2 / v1_safe + eps; - - for (size_t f = 0; f < features; ++f) { - float wsum = 0.0f; - for (size_t t = 0; t < T; ++t) { - wsum += static_cast(batch_src[f * T + t]) * batch_w[t]; - } - float mean = wsum / v1_safe; - - float wvar = 0.0f; - for (size_t t = 0; t < T; ++t) { - float dx = static_cast(batch_src[f * T + t]) - mean; - wvar += batch_w[t] * dx * dx; - } - float std_val = sqrtf(fmaxf(wvar / var_denom, 0.0f)); - - batch_dst[f] = static_cast<__fp16>(mean); - batch_dst[features + f] = static_cast<__fp16>(std_val); - } - } -} - diff --git a/cactus/graph/graph_ops_sample.cpp b/cactus/graph/graph_ops_sample.cpp deleted file mode 100644 index 26f6b621f..000000000 --- a/cactus/graph/graph_ops_sample.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include - -void compute_sample_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& logits_buffer = get_input(node, 0, nodes, node_index_map); - - float temperature = node.params.temperature; - float top_p = node.params.top_p; - float min_p = node.params.min_p; - float repetition_penalty = node.params.repetition_penalty; - size_t top_k = node.params.top_k; - size_t random_seed = node.params.random_seed; - - const float* bias_values = node.params.bias_values.empty() ? nullptr : node.params.bias_values.data(); - const uint32_t* bias_indices = node.params.bias_indices.empty() ? nullptr : node.params.bias_indices.data(); - size_t bias_count = node.params.bias_values.size(); - - if (logits_buffer.shape.size() != 2) { - throw std::runtime_error("Sample expects 2D logits tensor [seq_len, vocab_size]"); - } - - size_t seq_len = logits_buffer.shape[0]; - size_t vocab_size = logits_buffer.shape[1]; - size_t last_token_offset = (seq_len - 1) * vocab_size; - - if (logits_buffer.precision == Precision::FP16) { - const __fp16* logits_fp16 = logits_buffer.data_as<__fp16>(); - cactus_sample_f16_ex(logits_fp16 + last_token_offset, node.output_buffer.data_as(), - vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, - bias_values, bias_indices, bias_count); - } else { - const float* logits_fp32 = logits_buffer.data_as(); - cactus_sample_f32_ex(logits_fp32 + last_token_offset, node.output_buffer.data_as(), - vocab_size, temperature, top_p, min_p, repetition_penalty, top_k, random_seed, - bias_values, bias_indices, bias_count); - } -} - -void compute_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - if (input_buffer.shape.size() != 2) { - throw std::runtime_error("TopK currently only supports 2D tensors [batch, features]"); - } - - size_t k = node.params.top_k; - size_t batch_size = input_buffer.shape[0]; - size_t feature_size = input_buffer.shape[1]; - size_t block_size = batch_size * k; - - std::vector input_float(input_buffer.total_size); - if (input_buffer.precision == Precision::INT8) { - throw std::runtime_error("TopK currently does not support INT8 input"); - } else if (input_buffer.precision == Precision::FP16) { - const __fp16* input_fp16 = input_buffer.data_as<__fp16>(); - for (size_t i = 0; i < input_buffer.total_size; ++i) { - input_float[i] = static_cast(input_fp16[i]); - } - } else { - const float* input_fp32 = input_buffer.data_as(); - std::memcpy(input_float.data(), input_fp32, input_buffer.total_size * sizeof(float)); - } - - float* output = node.output_buffer.data_as(); - - for (size_t b = 0; b < batch_size; ++b) { - const float* row = input_float.data() + b * feature_size; - - std::vector> indexed_values(feature_size); - for (size_t i = 0; i < feature_size; ++i) { - indexed_values[i] = {i, row[i]}; - } - - std::partial_sort(indexed_values.begin(), - indexed_values.begin() + k, - indexed_values.end(), - [](const auto& a, const auto& b) { return a.second > b.second; }); - - float* idx_out_row = output + b * k; - float* val_out_row = output + block_size + b * k; - for (size_t i = 0; i < k; ++i) { - idx_out_row[i] = static_cast(indexed_values[i].first); - val_out_row[i] = indexed_values[i].second; - } - } -} - -void compute_scatter_topk_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& indices_buffer = get_input(node, 0, nodes, node_index_map); - const auto& values_buffer = get_input(node, 1, nodes, node_index_map); - - if (indices_buffer.shape != values_buffer.shape) { - throw std::runtime_error("ScatterTopK requires indices and values with identical shapes"); - } - if (indices_buffer.shape.size() != 2) { - throw std::runtime_error("ScatterTopK currently supports 2D tensors"); - } - - size_t batch_size = indices_buffer.shape[0]; - size_t top_k = indices_buffer.shape[1]; - size_t num_classes = node.params.num_classes; - - if (num_classes == 0) { - throw std::runtime_error("ScatterTopK requires num_classes > 0"); - } - - float* output = node.output_buffer.data_as(); - std::fill(output, output + num_classes * batch_size, 0.0f); - - if (indices_buffer.precision != Precision::FP32 || values_buffer.precision != Precision::FP32) { - throw std::runtime_error("ScatterTopK currently expects FP32 inputs"); - } - - const float* indices_data = indices_buffer.data_as(); - const float* values_data = values_buffer.data_as(); - - for (size_t b = 0; b < batch_size; ++b) { - for (size_t k = 0; k < top_k; ++k) { - float raw_index = indices_data[b * top_k + k]; - if (!std::isfinite(raw_index)) { - throw std::runtime_error("ScatterTopK index is not finite"); - } - size_t expert_index = static_cast(raw_index + 0.5f); - if (expert_index >= num_classes) { - throw std::runtime_error("ScatterTopK index out of range"); - } - float weight = values_data[b * top_k + k]; - output[expert_index * batch_size + b] = weight; - } - } -} diff --git a/cactus/graph/graph_ops_tensor.cpp b/cactus/graph/graph_ops_tensor.cpp deleted file mode 100644 index 8ccdcd49d..000000000 --- a/cactus/graph/graph_ops_tensor.cpp +++ /dev/null @@ -1,470 +0,0 @@ -#include "graph.h" -#include "../kernel/kernel.h" -#include "../kernel/kernel_utils.h" -#include -#include -#include - -void compute_transpose_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.params.backend == ComputeBackend::NPU) { - throw std::runtime_error("NPU transpose operation not yet implemented"); - } - - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - - if (input_buffer.precision != Precision::FP16) { - throw std::runtime_error("Transpose only supports FP16 precision"); - } - - const auto& permutation = node.params.permutation; - - const __fp16* input = input_buffer.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - cactus_transpose_f16(input, output, input_buffer.shape.data(), permutation.data(), permutation.size(), 0, input_buffer.total_size); -} - -void compute_gather_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& tensor_buffer = get_input(node, 0, nodes, node_index_map); - const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); - - size_t first_dim = tensor_buffer.shape[0]; - size_t element_size = 1; - for (size_t i = 1; i < tensor_buffer.shape.size(); i++) { - element_size *= tensor_buffer.shape[i]; - } - - size_t num_indices = indices_buffer.total_size; - size_t bytes_per_element = PrecisionTraits::packed_size_of(tensor_buffer.precision, element_size); - - if (PrecisionTraits::is_integer(tensor_buffer.precision)) { - const char* tensor_data = static_cast(tensor_buffer.get_data()); - char* output = static_cast(node.output_buffer.get_data()); - Precision prec = tensor_buffer.precision; - - const bool is_grouped = tensor_buffer.group_size > 0; - __fp16* gathered_scales = nullptr; - const __fp16* src_scales = nullptr; - size_t num_groups = 0; - - if (is_grouped) { - num_groups = tensor_buffer.num_groups; - src_scales = tensor_buffer.scales_as_fp16(); - size_t scales_bytes = num_indices * num_groups * sizeof(__fp16); - node.output_buffer.owned_scales = std::make_unique(scales_bytes); - gathered_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); - } - - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + PrecisionTraits::byte_offset_of(prec, i * element_size), - tensor_data + PrecisionTraits::byte_offset_of(prec, idx * element_size), - bytes_per_element); - if (is_grouped) { - for (size_t g = 0; g < num_groups; g++) { - gathered_scales[i * num_groups + g] = src_scales[idx * num_groups + g]; - } - } - } - - if (is_grouped) { - node.output_buffer.group_size = tensor_buffer.group_size; - node.output_buffer.num_groups = num_groups; - node.output_buffer.scales_data = gathered_scales; - } - } else if (tensor_buffer.precision == Precision::FP16) { - const __fp16* tensor_data = tensor_buffer.data_as<__fp16>(); - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (indices_buffer.precision == Precision::INT8) { - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } else { - const float* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } - } else { - const float* tensor_data = tensor_buffer.data_as(); - float* output = node.output_buffer.data_as(); - - if (indices_buffer.precision == Precision::INT8) { - const int8_t* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } else { - const float* indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices[i]); - if (idx >= first_dim) { - throw std::runtime_error("Gather index " + std::to_string(idx) + " out of bounds for dimension " + std::to_string(first_dim)); - } - std::memcpy(output + i * element_size, tensor_data + idx * element_size, bytes_per_element); - } - } - } -} - -void compute_slice_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - auto* input_node = nodes[node_index_map.at(node.input_ids[0])].get(); - auto& input_buffer = input_node->output_buffer; - - const size_t axis_index = static_cast(node.params.axis); - - const size_t axis_size = input_buffer.shape[axis_index]; - const size_t slice_start = node.params.slice_start; - size_t slice_length = node.params.slice_length; - - if (slice_length == 0) { - slice_length = axis_size - slice_start; - } - - if (axis_index == 0) { - auto dims0 = AxisDims::from_shape(input_buffer.shape, 0); - - auto* base_ptr = static_cast(input_buffer.get_data()); - if (!base_ptr) { - throw std::runtime_error("Slice input buffer is not available"); - } - - const size_t byte_offset = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims0.inner); - - node.output_buffer.set_external(base_ptr + byte_offset); - node.output_buffer.precision = input_buffer.precision; - - if (input_buffer.is_grouped_int8()) { - size_t num_groups = input_buffer.num_groups; - size_t scales_bytes = slice_length * num_groups * sizeof(__fp16); - node.output_buffer.owned_scales = std::make_unique(scales_bytes); - __fp16* sliced_scales = reinterpret_cast<__fp16*>(node.output_buffer.owned_scales.get()); - const __fp16* input_scales = input_buffer.scales_as_fp16(); - - for (size_t i = 0; i < slice_length; i++) { - for (size_t g = 0; g < num_groups; g++) { - sliced_scales[i * num_groups + g] = input_scales[(slice_start + i) * num_groups + g]; - } - } - - node.output_buffer.group_size = input_buffer.group_size; - node.output_buffer.num_groups = num_groups; - node.output_buffer.scales_data = sliced_scales; - } - return; - } - - const char* input_ptr = static_cast(input_buffer.get_data()); - if (!input_ptr) { - throw std::runtime_error("Slice input buffer is not available"); - } - - auto dims = AxisDims::from_shape(input_buffer.shape, axis_index); - - node.output_buffer.external_data = nullptr; - node.output_buffer.allocate(); - node.output_buffer.precision = input_buffer.precision; - - auto* output_ptr = static_cast(node.output_buffer.get_data()); - if (!output_ptr) { - throw std::runtime_error("Slice output buffer could not be allocated"); - } - - const size_t copy_block_elements = slice_length * dims.inner; - const size_t axis_stride_elements = axis_size * dims.inner; - const size_t copy_block_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, copy_block_elements); - const size_t axis_stride_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, axis_stride_elements); - - for (size_t outer = 0; outer < dims.outer; ++outer) { - const char* src = input_ptr + outer * axis_stride_bytes + PrecisionTraits::byte_offset_of(input_buffer.precision, slice_start * dims.inner); - char* dst = output_ptr + outer * copy_block_bytes; - std::memcpy(dst, src, copy_block_bytes); - } -} - -void compute_embedding_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& embeddings_buffer = get_input(node, 0, nodes, node_index_map); - const auto& indices_buffer = get_input(node, 1, nodes, node_index_map); - - size_t hidden_dim = embeddings_buffer.shape[1]; - size_t num_indices = indices_buffer.total_size; - size_t vocab_size = embeddings_buffer.original_N > 0 - ? embeddings_buffer.original_N - : embeddings_buffer.shape[0]; - - std::vector indices_float; - const float* indices_ptr; - if (indices_buffer.precision == Precision::FP32) { - indices_ptr = indices_buffer.data_as(); - } else { - indices_float.resize(num_indices); - const int8_t* int_indices = indices_buffer.data_as(); - for (size_t i = 0; i < num_indices; i++) { - indices_float[i] = static_cast(int_indices[i]); - } - indices_ptr = indices_float.data(); - } - - __fp16* output = node.output_buffer.data_as<__fp16>(); - - Precision emb_prec = embeddings_buffer.precision; - if (PrecisionTraits::is_integer(emb_prec) && embeddings_buffer.group_size > 0) { - const int8_t* embeddings = embeddings_buffer.data_as(); - const __fp16* scales = embeddings_buffer.scales_as_fp16(); - size_t group_size = embeddings_buffer.group_size; - size_t num_groups = embeddings_buffer.num_groups; - - static const uint8_t gather_indices[4][16] = { - {0, 1, 2, 3, 16, 17, 18, 19, 32, 33, 34, 35, 48, 49, 50, 51}, // lane 0 - {4, 5, 6, 7, 20, 21, 22, 23, 36, 37, 38, 39, 52, 53, 54, 55}, // lane 1 - {8, 9, 10, 11, 24, 25, 26, 27, 40, 41, 42, 43, 56, 57, 58, 59}, // lane 2 - {12, 13, 14, 15, 28, 29, 30, 31, 44, 45, 46, 47, 60, 61, 62, 63} // lane 3 - }; - - auto load_table = [emb_prec](const int8_t* base) -> int8x16x4_t { - if (emb_prec == Precision::INT4) { - const uint8_t* ubase = reinterpret_cast(base); - int8x16_t low_a, high_a, low_b, high_b; - unpack_int4_as_int8x16x2(ubase, high_a, low_a); - unpack_int4_as_int8x16x2(ubase + 16, high_b, low_b); - return {low_a, high_a, low_b, high_b}; - } - return vld1q_s8_x4(base); - }; - - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices_ptr[i]); - if (idx >= vocab_size) { - throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); - } - - size_t block = idx / 4; - size_t lane = idx % 4; - __fp16* out_row = output + i * hidden_dim; - - uint8x16_t indices_vec = vld1q_u8(gather_indices[lane]); - - for (size_t g = 0; g < num_groups; g++) { - float scale = (float)scales[(block * num_groups + g) * 4 + lane]; - float32x4_t scale_vec = vdupq_n_f32(scale); - - size_t k_start = g * group_size; - size_t k_end = std::min(k_start + group_size, hidden_dim); - - const int8_t* group_base = embeddings + PrecisionTraits::byte_offset_of(emb_prec, (block * hidden_dim + k_start) * 4); - - size_t k = k_start; - for (; k + 16 <= k_end; k += 16) { - const int8_t* chunk_base = group_base + PrecisionTraits::byte_offset_of(emb_prec, (k - k_start) * 4); - int8x16x4_t table = load_table(chunk_base); - - int8x16_t values = vqtbl4q_s8(table, indices_vec); - - int16x8_t lo16 = vmovl_s8(vget_low_s8(values)); - int16x8_t hi16 = vmovl_s8(vget_high_s8(values)); - - float32x4_t f0 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(lo16))), scale_vec); - float32x4_t f1 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(lo16))), scale_vec); - float32x4_t f2 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(hi16))), scale_vec); - float32x4_t f3 = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(hi16))), scale_vec); - - vst1q_f16(out_row + k, vcombine_f16(vcvt_f16_f32(f0), vcvt_f16_f32(f1))); - vst1q_f16(out_row + k + 8, vcombine_f16(vcvt_f16_f32(f2), vcvt_f16_f32(f3))); - } - - if (k < k_end) - assert (embeddings_buffer.precision != Precision::INT4 && "grouped INT4 embeddings must have hidden_dim that is a multiple of 32"); - for (; k < k_end; k++) { - size_t k_group = k / 4; - size_t k_within = k % 4; - int8_t val = embeddings[(block * (hidden_dim / 4) + k_group) * 16 + lane * 4 + k_within]; - out_row[k] = static_cast<__fp16>(val * scale); - } - } - } - } else if (embeddings_buffer.precision == Precision::FP16) { - const __fp16* embeddings = embeddings_buffer.data_as<__fp16>(); - for (size_t i = 0; i < num_indices; i++) { - size_t idx = static_cast(indices_ptr[i]); - if (idx >= vocab_size) { - throw std::runtime_error("Embedding index out of bounds: " + std::to_string(idx) + " >= " + std::to_string(vocab_size)); - } - std::memcpy(output + i * hidden_dim, embeddings + idx * hidden_dim, hidden_dim * sizeof(__fp16)); - } - } else { - throw std::runtime_error("Embedding requires interleaved grouped INT4/INT8 or FP16"); - } -} - -void compute_concat_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input1_buffer = get_input(node, 0, nodes, node_index_map); - const auto& input2_buffer = get_input(node, 1, nodes, node_index_map); - - std::vector shape1 = input1_buffer.shape; - std::vector shape2 = input2_buffer.shape; - std::vector output_shape = node.output_buffer.shape; - - if (input1_buffer.precision != Precision::FP16) { - throw std::runtime_error("Concat operation only supports FP16 precision"); - } - cactus_concat_f16(input1_buffer.data_as<__fp16>(), input2_buffer.data_as<__fp16>(), - node.output_buffer.data_as<__fp16>(), - shape1.data(), shape2.data(), output_shape.data(), - shape1.size(), node.params.axis); -} - -void compute_cat_node( - GraphNode& node, - const std::vector>& nodes, - const std::unordered_map& node_index_map -) { - if (node.params.axis < 0) { - throw std::runtime_error("Cat operation does not support negative axis"); - } - if (node.input_ids.size() < 2) { - throw std::runtime_error("Cat operation requires at least 2 input tensors"); - } - - const auto& first_buffer = get_input(node, 0, nodes, node_index_map); - - if (first_buffer.precision != Precision::FP16) { - throw std::runtime_error("Cat operation only supports FP16 precision"); - } - - std::vector input_data_ptrs(node.input_ids.size()); - std::vector input_shape_ptrs(node.input_ids.size()); - - for (size_t i = 0; i < node.input_ids.size(); i++) { - const auto& buffer = get_input(node, i, nodes, node_index_map); - - if (buffer.precision != Precision::FP16) { - throw std::runtime_error("Cat operation only supports FP16 precision"); - } - - input_data_ptrs[i] = buffer.data_as<__fp16>(); - input_shape_ptrs[i] = buffer.shape.data(); - } - - cactus_cat_f16(input_data_ptrs.data(), - node.output_buffer.data_as<__fp16>(), - input_shape_ptrs.data(), - node.output_buffer.shape.data(), - node.input_ids.size(), - node.output_buffer.shape.size(), - node.params.axis); -} - -void compute_index_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& input_buffer = get_input(node, 0, nodes, node_index_map); - const auto& input_shape = input_buffer.shape; - - int dim = node.params.axis; - size_t index_value = node.params.index_value; - - const char* input_data = static_cast(input_buffer.get_data()); - char* output_data = static_cast(node.output_buffer.get_data()); - - if (dim == 0) { - size_t slice_size = input_buffer.total_size / input_shape[0]; - size_t offset_bytes = PrecisionTraits::byte_offset_of(input_buffer.precision, index_value * slice_size); - node.output_buffer.set_external(const_cast(input_data) + offset_bytes); - return; - } - - std::vector input_strides(input_shape.size()); - input_strides[input_shape.size() - 1] = 1; - for (int i = static_cast(input_shape.size()) - 2; i >= 0; --i) { - input_strides[i] = input_strides[i + 1] * input_shape[i + 1]; - } - - size_t slice_size = input_strides[dim]; - size_t outer_size = input_buffer.total_size / input_strides[dim - 1]; - size_t dim_stride = input_strides[dim]; - size_t block_size = dim_stride * input_shape[dim]; - - size_t output_idx = 0; - for (size_t outer_idx = 0; outer_idx < outer_size; ++outer_idx) { - size_t input_base = outer_idx * block_size + index_value * dim_stride; - - char* output_offset_bytes = output_data + PrecisionTraits::byte_offset_of(input_buffer.precision, output_idx); - const char* input_offset_bytes = input_data + PrecisionTraits::byte_offset_of(input_buffer.precision, input_base); - size_t length = PrecisionTraits::byte_offset_of(input_buffer.precision, slice_size); - std::memcpy(output_offset_bytes, input_offset_bytes, length); - - output_idx += slice_size; - } -} - -void compute_bilinear_interpolation_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - const auto& pos_embeds_buffer = get_input(node, 0, nodes, node_index_map); - - size_t total_pos_embeds = pos_embeds_buffer.shape[0]; - size_t embed_dim = pos_embeds_buffer.shape[1]; - - size_t src_height = static_cast(std::sqrt(total_pos_embeds)); - size_t src_width = src_height; - - size_t dst_height = node.params.dst_height; - size_t dst_width = node.params.dst_width; - bool align_corners = node.params.align_corners; - - __fp16* output = node.output_buffer.data_as<__fp16>(); - - if (pos_embeds_buffer.precision == Precision::FP16) { - const __fp16* input = pos_embeds_buffer.data_as<__fp16>(); - cactus_bilinear_interpolation_f16(input, output, src_height, src_width, embed_dim, - dst_height, dst_width, align_corners); - } - else if (pos_embeds_buffer.precision == Precision::INT8) { - std::vector<__fp16> input_fp16(total_pos_embeds * embed_dim); - cactus_int8_to_fp16(pos_embeds_buffer.data_as(), input_fp16.data(), - total_pos_embeds * embed_dim); - cactus_bilinear_interpolation_f16(input_fp16.data(), output, src_height, src_width, embed_dim, - dst_height, dst_width, align_corners); - } - else { - throw std::runtime_error("BILINEAR_INTERPOLATION only supports INT8 and FP16 input precision"); - } -} - -void compute_persistent_node(GraphNode& node, const std::vector>& nodes, const std::unordered_map& node_index_map) { - if (node.input_ids.empty()) { - return; - } - - auto it = node_index_map.find(node.input_ids[0]); - - if (it != node_index_map.end()) { - const auto& input_buffer = nodes[it->second]->output_buffer; - - if (!node.output_buffer.get_data()) { - node.output_buffer.allocate(); - } - - std::memcpy(node.output_buffer.get_data(), - input_buffer.get_data(), - input_buffer.byte_size); - } else { - if (node.output_buffer.get_data()) { - return; - } - throw std::runtime_error("PERSISTENT node input not found and not populated - this should not happen"); - } -} diff --git a/cactus/graph/graph_param_io.cpp b/cactus/graph/graph_param_io.cpp deleted file mode 100644 index 6d49c9ad6..000000000 --- a/cactus/graph/graph_param_io.cpp +++ /dev/null @@ -1,380 +0,0 @@ -#include "graph_param_io.h" - -#include -#include -#include -#include - -namespace { - -void write_u32(std::ostream& out, uint32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_u64(std::ostream& out, uint64_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_i32(std::ostream& out, int32_t v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -void write_f32(std::ostream& out, float v) { - out.write(reinterpret_cast(&v), sizeof(v)); -} - -uint32_t read_u32(std::istream& in) { - uint32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading uint32"); - return v; -} - -uint64_t read_u64(std::istream& in) { - uint64_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading uint64"); - return v; -} - -int32_t read_i32(std::istream& in) { - int32_t v = 0; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading int32"); - return v; -} - -float read_f32(std::istream& in) { - float v = 0.0f; - in.read(reinterpret_cast(&v), sizeof(v)); - if (!in) throw std::runtime_error("Unexpected EOF while reading float"); - return v; -} - -void write_size_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (size_t v : values) write_u64(out, static_cast(v)); -} - -std::vector read_size_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(static_cast(read_u64(in))); - return values; -} - -void write_u32_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (uint32_t v : values) write_u32(out, v); -} - -std::vector read_u32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(read_u32(in)); - return values; -} - -void write_f32_vector(std::ostream& out, const std::vector& values) { - write_u32(out, static_cast(values.size())); - for (float v : values) write_f32(out, v); -} - -std::vector read_f32_vector(std::istream& in) { - uint32_t count = read_u32(in); - std::vector values; - values.reserve(count); - for (uint32_t i = 0; i < count; ++i) values.push_back(read_f32(in)); - return values; -} - -enum class ParamField : uint32_t { - Scalar = 1, - Axis, - NewShape, - PretransposedRhs, - Backend, - SliceStart, - SliceLength, - Epsilon, - NumGroups, - IndexValue, - Permutation, - Scale, - Theta, - PositionOffset, - WindowSize, - IsCausal, - AttentionMaskIsAdditive, - LogitCap, - TopK, - Temperature, - TopP, - MinP, - RepetitionPenalty, - RandomSeed, - BiasIndices, - BiasValues, - NormalizeRouting, - NumExperts, - NumExpertsPerTok, - MoeGated, - Activation, - NumKvHeads, - ChunkSize, - DstHeight, - DstWidth, - AlignCorners, - NumClasses, - NumAltupInputs, - KernelSize, - Stride, - Dilation, - NumFftBins, - CacheSeqLen, - HeadDim, - VHeadDim, - CachedKeysInt8Ptr, - CachedValuesInt8Ptr, - CachedKScalesPtr, - CachedVScalesPtr, -}; - -enum class FieldPersistence { - Persistent, - RuntimeOnly, - Derived, -}; - -struct FieldSpec { - ParamField field; - FieldPersistence persistence; -}; - -using Schema = std::vector; - -const Schema& op_schema(OpType op_type) { - static const Schema kEmpty{}; - static const std::unordered_map kSchemas = { - {OpType::POW, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_ADD, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_SUBTRACT, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_MULTIPLY, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SCALAR_DIVIDE, {{ParamField::Scalar, FieldPersistence::Persistent}}}, - {OpType::SOFTMAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::SUM, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MEAN, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::VARIANCE, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MIN, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::MAX, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::INDEX, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::IndexValue, FieldPersistence::Persistent}}}, - {OpType::CONCAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::CAT, {{ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::VIEW, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::RESHAPE, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::FLATTEN, {{ParamField::NewShape, FieldPersistence::Persistent}}}, - {OpType::MATMUL, {{ParamField::PretransposedRhs, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::TRANSPOSE, {{ParamField::Permutation, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::SLICE, {{ParamField::Axis, FieldPersistence::Persistent}, {ParamField::SliceStart, FieldPersistence::Persistent}, {ParamField::SliceLength, FieldPersistence::Persistent}}}, - {OpType::RMS_NORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, - {OpType::LAYERNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}}}, - {OpType::GROUPNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NumGroups, FieldPersistence::Persistent}}}, - {OpType::BATCHNORM, {{ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::Axis, FieldPersistence::Persistent}}}, - {OpType::ROPE, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::ROPE_GPTJ, {{ParamField::Theta, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::TOPK, {{ParamField::TopK, FieldPersistence::Persistent}}}, - {OpType::ATTENTION, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::PositionOffset, FieldPersistence::Persistent}, {ParamField::WindowSize, FieldPersistence::Persistent}, {ParamField::IsCausal, FieldPersistence::Persistent}, {ParamField::AttentionMaskIsAdditive, FieldPersistence::Persistent}, {ParamField::LogitCap, FieldPersistence::Persistent}, {ParamField::Backend, FieldPersistence::Persistent}}}, - {OpType::REL_POS_BIAS, {{ParamField::Scale, FieldPersistence::Persistent}}}, - {OpType::ATTENTION_INT8_HYBRID, { - {ParamField::Scale, FieldPersistence::Persistent}, - {ParamField::PositionOffset, FieldPersistence::Persistent}, - {ParamField::WindowSize, FieldPersistence::Persistent}, - {ParamField::NumKvHeads, FieldPersistence::Persistent}, - {ParamField::CacheSeqLen, FieldPersistence::Persistent}, - {ParamField::HeadDim, FieldPersistence::Persistent}, - {ParamField::VHeadDim, FieldPersistence::Persistent}, - {ParamField::CachedKeysInt8Ptr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedValuesInt8Ptr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedKScalesPtr, FieldPersistence::RuntimeOnly}, - {ParamField::CachedVScalesPtr, FieldPersistence::RuntimeOnly}, - }}, - {OpType::MOE_LAYER, {{ParamField::Scalar, FieldPersistence::Persistent}, {ParamField::Epsilon, FieldPersistence::Persistent}, {ParamField::NormalizeRouting, FieldPersistence::Persistent}, {ParamField::NumExperts, FieldPersistence::Persistent}, {ParamField::NumExpertsPerTok, FieldPersistence::Persistent}, {ParamField::MoeGated, FieldPersistence::Persistent}, {ParamField::Activation, FieldPersistence::Persistent}}}, - {OpType::GATED_DELTANET_DECODE, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}}}, - {OpType::GATED_DELTANET_PREFILL, {{ParamField::Scale, FieldPersistence::Persistent}, {ParamField::NumKvHeads, FieldPersistence::Persistent}, {ParamField::ChunkSize, FieldPersistence::Persistent}}}, - {OpType::STFT, {{ParamField::Stride, FieldPersistence::Persistent}, {ParamField::NumFftBins, FieldPersistence::Persistent}}}, - {OpType::BILINEAR_INTERPOLATION, {{ParamField::DstHeight, FieldPersistence::Persistent}, {ParamField::DstWidth, FieldPersistence::Persistent}, {ParamField::AlignCorners, FieldPersistence::Persistent}}}, - {OpType::SCATTER_TOPK, {{ParamField::NumClasses, FieldPersistence::Persistent}}}, - {OpType::ALTUP_PREDICT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, - {OpType::ALTUP_CORRECT, {{ParamField::NumAltupInputs, FieldPersistence::Persistent}}}, - {OpType::MAXPOOL1D, {{ParamField::KernelSize, FieldPersistence::Persistent}, {ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D_CAUSAL, {{ParamField::Dilation, FieldPersistence::Persistent}}}, - {OpType::CONV1D_K3, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D_K7S3, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::CONV1D, {{ParamField::Stride, FieldPersistence::Persistent}}}, - {OpType::SAMPLE, {{ParamField::Temperature, FieldPersistence::Persistent}, {ParamField::TopP, FieldPersistence::Persistent}, {ParamField::MinP, FieldPersistence::Persistent}, {ParamField::RepetitionPenalty, FieldPersistence::Persistent}, {ParamField::TopK, FieldPersistence::Persistent}, {ParamField::RandomSeed, FieldPersistence::Persistent}, {ParamField::BiasIndices, FieldPersistence::Persistent}, {ParamField::BiasValues, FieldPersistence::Persistent}}}, - }; - - auto it = kSchemas.find(op_type); - return it == kSchemas.end() ? kEmpty : it->second; -} - -void write_field(std::ostream& out, ParamField field, const OpParams& params) { - switch (field) { - case ParamField::Scalar: write_f32(out, params.scalar); break; - case ParamField::Axis: write_i32(out, static_cast(params.axis)); break; - case ParamField::NewShape: write_size_vector(out, params.new_shape); break; - case ParamField::PretransposedRhs: write_u32(out, params.pretransposed_rhs ? 1u : 0u); break; - case ParamField::Backend: write_u32(out, static_cast(params.backend)); break; - case ParamField::SliceStart: write_u64(out, static_cast(params.slice_start)); break; - case ParamField::SliceLength: write_u64(out, static_cast(params.slice_length)); break; - case ParamField::Epsilon: write_f32(out, params.epsilon); break; - case ParamField::NumGroups: write_u64(out, static_cast(params.num_groups)); break; - case ParamField::IndexValue: write_u64(out, static_cast(params.index_value)); break; - case ParamField::Permutation: write_size_vector(out, params.permutation); break; - case ParamField::Scale: write_f32(out, params.scale); break; - case ParamField::Theta: write_f32(out, params.theta); break; - case ParamField::PositionOffset: write_u64(out, static_cast(params.position_offset)); break; - case ParamField::WindowSize: write_u64(out, static_cast(params.window_size)); break; - case ParamField::IsCausal: write_u32(out, params.is_causal ? 1u : 0u); break; - case ParamField::AttentionMaskIsAdditive: write_u32(out, params.attention_mask_is_additive ? 1u : 0u); break; - case ParamField::LogitCap: write_f32(out, params.logit_cap); break; - case ParamField::TopK: write_u64(out, static_cast(params.top_k)); break; - case ParamField::Temperature: write_f32(out, params.temperature); break; - case ParamField::TopP: write_f32(out, params.top_p); break; - case ParamField::MinP: write_f32(out, params.min_p); break; - case ParamField::RepetitionPenalty: write_f32(out, params.repetition_penalty); break; - case ParamField::RandomSeed: write_u64(out, static_cast(params.random_seed)); break; - case ParamField::BiasIndices: write_u32_vector(out, params.bias_indices); break; - case ParamField::BiasValues: write_f32_vector(out, params.bias_values); break; - case ParamField::NormalizeRouting: write_u32(out, params.normalize_routing ? 1u : 0u); break; - case ParamField::NumExperts: write_u64(out, static_cast(params.num_experts)); break; - case ParamField::NumExpertsPerTok: write_u64(out, static_cast(params.num_experts_per_tok)); break; - case ParamField::MoeGated: write_u32(out, params.moe_gated ? 1u : 0u); break; - case ParamField::Activation: write_u32(out, static_cast(params.activation)); break; - case ParamField::NumKvHeads: write_u64(out, static_cast(params.num_kv_heads)); break; - case ParamField::ChunkSize: write_u64(out, static_cast(params.chunk_size)); break; - case ParamField::DstHeight: write_u64(out, static_cast(params.dst_height)); break; - case ParamField::DstWidth: write_u64(out, static_cast(params.dst_width)); break; - case ParamField::AlignCorners: write_u32(out, params.align_corners ? 1u : 0u); break; - case ParamField::NumClasses: write_u64(out, static_cast(params.num_classes)); break; - case ParamField::NumAltupInputs: write_u64(out, static_cast(params.num_altup_inputs)); break; - case ParamField::KernelSize: write_u64(out, static_cast(params.kernel_size)); break; - case ParamField::Stride: write_u64(out, static_cast(params.stride)); break; - case ParamField::Dilation: write_u64(out, static_cast(params.dilation)); break; - case ParamField::NumFftBins: write_u64(out, static_cast(params.num_fft_bins)); break; - case ParamField::CacheSeqLen: write_u64(out, static_cast(params.cache_seq_len)); break; - case ParamField::HeadDim: write_u64(out, static_cast(params.head_dim)); break; - case ParamField::VHeadDim: write_u64(out, static_cast(params.v_head_dim)); break; - case ParamField::CachedKeysInt8Ptr: - case ParamField::CachedValuesInt8Ptr: - case ParamField::CachedKScalesPtr: - case ParamField::CachedVScalesPtr: - throw std::runtime_error("Attempted to serialize runtime-only pointer field"); - } -} - -void read_field(std::istream& in, ParamField field, OpParams& params) { - switch (field) { - case ParamField::Scalar: params.scalar = read_f32(in); break; - case ParamField::Axis: params.axis = static_cast(read_i32(in)); break; - case ParamField::NewShape: params.new_shape = read_size_vector(in); break; - case ParamField::PretransposedRhs: params.pretransposed_rhs = (read_u32(in) != 0); break; - case ParamField::Backend: { - uint32_t backend_val = read_u32(in); - if (backend_val > static_cast(ComputeBackend::NPU)) { - throw std::runtime_error("Graph file corrupted: invalid backend"); - } - params.backend = static_cast(backend_val); - break; - } - case ParamField::SliceStart: params.slice_start = static_cast(read_u64(in)); break; - case ParamField::SliceLength: params.slice_length = static_cast(read_u64(in)); break; - case ParamField::Epsilon: params.epsilon = read_f32(in); break; - case ParamField::NumGroups: params.num_groups = static_cast(read_u64(in)); break; - case ParamField::IndexValue: params.index_value = static_cast(read_u64(in)); break; - case ParamField::Permutation: params.permutation = read_size_vector(in); break; - case ParamField::Scale: params.scale = read_f32(in); break; - case ParamField::Theta: params.theta = read_f32(in); break; - case ParamField::PositionOffset: params.position_offset = static_cast(read_u64(in)); break; - case ParamField::WindowSize: params.window_size = static_cast(read_u64(in)); break; - case ParamField::IsCausal: params.is_causal = (read_u32(in) != 0); break; - case ParamField::AttentionMaskIsAdditive: params.attention_mask_is_additive = (read_u32(in) != 0); break; - case ParamField::LogitCap: params.logit_cap = read_f32(in); break; - case ParamField::TopK: params.top_k = static_cast(read_u64(in)); break; - case ParamField::Temperature: params.temperature = read_f32(in); break; - case ParamField::TopP: params.top_p = read_f32(in); break; - case ParamField::MinP: params.min_p = read_f32(in); break; - case ParamField::RepetitionPenalty: params.repetition_penalty = read_f32(in); break; - case ParamField::RandomSeed: params.random_seed = static_cast(read_u64(in)); break; - case ParamField::BiasIndices: params.bias_indices = read_u32_vector(in); break; - case ParamField::BiasValues: params.bias_values = read_f32_vector(in); break; - case ParamField::NormalizeRouting: params.normalize_routing = (read_u32(in) != 0); break; - case ParamField::NumExperts: params.num_experts = static_cast(read_u64(in)); break; - case ParamField::NumExpertsPerTok: params.num_experts_per_tok = static_cast(read_u64(in)); break; - case ParamField::MoeGated: params.moe_gated = (read_u32(in) != 0); break; - case ParamField::Activation: params.activation = static_cast(read_u32(in)); break; - case ParamField::NumKvHeads: params.num_kv_heads = static_cast(read_u64(in)); break; - case ParamField::ChunkSize: params.chunk_size = static_cast(read_u64(in)); break; - case ParamField::DstHeight: params.dst_height = static_cast(read_u64(in)); break; - case ParamField::DstWidth: params.dst_width = static_cast(read_u64(in)); break; - case ParamField::AlignCorners: params.align_corners = (read_u32(in) != 0); break; - case ParamField::NumClasses: params.num_classes = static_cast(read_u64(in)); break; - case ParamField::NumAltupInputs: params.num_altup_inputs = static_cast(read_u64(in)); break; - case ParamField::KernelSize: params.kernel_size = static_cast(read_u64(in)); break; - case ParamField::Stride: params.stride = static_cast(read_u64(in)); break; - case ParamField::Dilation: params.dilation = static_cast(read_u64(in)); break; - case ParamField::NumFftBins: params.num_fft_bins = static_cast(read_u64(in)); break; - case ParamField::CacheSeqLen: params.cache_seq_len = static_cast(read_u64(in)); break; - case ParamField::HeadDim: params.head_dim = static_cast(read_u64(in)); break; - case ParamField::VHeadDim: params.v_head_dim = static_cast(read_u64(in)); break; - case ParamField::CachedKeysInt8Ptr: - case ParamField::CachedValuesInt8Ptr: - case ParamField::CachedKScalesPtr: - case ParamField::CachedVScalesPtr: - throw std::runtime_error("Graph file corrupted: runtime-only field serialized"); - } -} - -} // namespace - -namespace GraphParamIO { - -void write_op_params(std::ostream& out, OpType op_type, const OpParams& params) { - const auto& schema = op_schema(op_type); - std::vector fields; - fields.reserve(schema.size()); - for (const auto& spec : schema) { - if (spec.persistence == FieldPersistence::Persistent) { - fields.push_back(spec.field); - } - } - - write_u32(out, static_cast(fields.size())); - for (ParamField field : fields) { - write_u32(out, static_cast(field)); - write_field(out, field, params); - } -} - -void read_op_params(std::istream& in, OpType op_type, OpParams& params) { - uint32_t field_count = read_u32(in); - const auto& schema = op_schema(op_type); - std::unordered_set allowed; - for (const auto& spec : schema) { - if (spec.persistence == FieldPersistence::Persistent) { - allowed.insert(static_cast(spec.field)); - } - } - - for (uint32_t i = 0; i < field_count; ++i) { - uint32_t raw_field = read_u32(in); - if (allowed.count(raw_field) == 0) { - throw std::runtime_error("Graph file corrupted: field not allowed for op"); - } - read_field(in, static_cast(raw_field), params); - } -} - -} // namespace GraphParamIO diff --git a/cactus/graph/graph_param_io.h b/cactus/graph/graph_param_io.h deleted file mode 100644 index 0362ea8c5..000000000 --- a/cactus/graph/graph_param_io.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#include "graph.h" -#include - -namespace GraphParamIO { - -void write_op_params(std::ostream& out, OpType op_type, const OpParams& params); -void read_op_params(std::istream& in, OpType op_type, OpParams& params); - -} // namespace GraphParamIO diff --git a/cactus/kernel/kernel.h b/cactus/kernel/kernel.h deleted file mode 100644 index 13f4a00be..000000000 --- a/cactus/kernel/kernel.h +++ /dev/null @@ -1,475 +0,0 @@ -#ifndef KERNEL_H -#define KERNEL_H - -#include -#include - -enum class Precision; - -enum class ScalarOpType { - ADD, - SUBTRACT, - MULTIPLY, - DIVIDE, - ABS, - EXP, - POW, - SQRT, - COS, - SIN, - LOG -}; - -constexpr size_t KV_QUANT_GROUP_SIZE = 32; - -void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); -void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale); -void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements); - -void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); -void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim); - -void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type); - -void cactus_gemv_int8(const int8_t* A, float A_scale, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemm_int8(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_int8(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_gemv_int8_i8mm(const int8_t* A, float A_scale, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemm_int8_i8mm(const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_gemv_int4(const int8_t* A, float A_scale, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size); - -void cactus_gemv_int4_block_range(const int8_t* A, float A_scale, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t K, size_t N, size_t group_size, - size_t block_start, size_t block_end); - -void cactus_gemm_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_integer(Precision precision, - const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size); - -void cactus_matmul_f16(const __fp16* a, const __fp16* b_transposed, __fp16* c, - size_t M, size_t K, size_t N); - -void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, - size_t num_rows, size_t num_cols, size_t start_row, size_t end_row); -void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, - const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx); - -double cactus_sum_all_f16(const __fp16* data, size_t num_elements); -void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); - -double cactus_mean_all_f16(const __fp16* data, size_t num_elements); -void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); - -double cactus_variance_all_f16(const __fp16* data, size_t num_elements); -void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); - -__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements); -void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); - -__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements); -void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size); - -void cactus_rms_norm_f16(const __fp16* input, const __fp16* weight, __fp16* output, - size_t batch_size, size_t dims, float eps); - -void cactus_layer_norm_f16(const __fp16* input, const __fp16* weight, const __fp16* bias, - __fp16* output, size_t batch_size, size_t dims, float eps); - -void cactus_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, - size_t num_heads, size_t head_dim, size_t start_pos, float theta); - -void cactus_gpt_j_rope_f16(const __fp16* input, __fp16* output, size_t batch_size, size_t seq_len, - size_t num_heads, size_t head_dim, size_t rot_dim, size_t start_pos, float theta); - -void cactus_softmax_f16(const __fp16* input, __fp16* output, size_t batch_size, - size_t seq_len, size_t vocab_size); - -void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope); - -void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements); - -void cactus_glu_f16( - const __fp16* input, - __fp16* output, - size_t outer_size, - size_t split_size, - size_t inner_size -); - -void cactus_glu_f32( - const float* input, - float* output, - size_t outer_size, - size_t split_size, - size_t inner_size -); - -void cactus_batchnorm_f16( - const __fp16* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - __fp16* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -); - -void cactus_batchnorm_f32( - const float* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - float* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -); - -void cactus_attention_f16(const __fp16* queries, const __fp16* keys, const __fp16* values, __fp16* output, - size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, - size_t head_dim, float scale, const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, - bool is_causal = true, bool mask_is_additive = false, bool mask_per_head = false, - size_t v_head_dim = 0, float logit_cap = 0.0f); - -void cactus_attention_hybrid_int8_fp16( - const __fp16* queries, - const int8_t* keys_cached, - const int8_t* values_cached, - const float* k_scales, - const float* v_scales, - const __fp16* keys_new, - const __fp16* values_new, - __fp16* output, - size_t batch_size, size_t seq_len, size_t cache_len, size_t new_len, - size_t num_q_heads, size_t num_kv_heads, size_t head_dim, - float scale, size_t position_offset = 0, bool is_causal = true, size_t window_size = 0, - size_t group_size = KV_QUANT_GROUP_SIZE, size_t v_head_dim = 0); - -void cactus_gated_deltanet_decode_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale); - -void cactus_gated_deltanet_prefill_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t requested_chunk_size, - float scale); - -void cactus_conv1d_causal_depthwise_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, - size_t L, - size_t C, - size_t K, - size_t dilation); - -void cactus_conv1d_f16_k3( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out, - size_t stride -); - -void cactus_conv1d_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out, - size_t K, - size_t stride -); - -void cactus_stft_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, size_t stride, - size_t num_fft_bins -); - -void cactus_conv1d_f16_k7s3_oc8( - const __fp16* input, - const __fp16* Wpack, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -); - -void cactus_conv1d_same_depthwise_f16_k9( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C -); - -void cactus_conv2d_f16_k3s1p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv2d_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv2d_depthwise_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C, - size_t H, - size_t W -); - -void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -); - -void cactus_conv1d_pointwise_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -); - -void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, - size_t dst_height, size_t dst_width, bool align_corners = true); - -void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); -void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); - -void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); -void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values = nullptr, const uint32_t* bias_indices = nullptr, - size_t bias_count = 0); - -void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, - const size_t* shape1, const size_t* shape2, const size_t* output_shape, - size_t ndims, int axis); -void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, - const size_t* output_shape, size_t num_inputs, size_t rank, int axis); - -void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale = 1.0f); -void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale = 1.0f); -void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count); -void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count); -void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale = 1.0f); -void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale = 1.0f); -float cactus_fp16_max_abs(const __fp16* src, size_t count); - -void cactus_quantize_kv_fp16_to_int8( - const __fp16* src, - int8_t* dst, - float* scales, - size_t seq_len, size_t kv_heads, size_t head_dim, - size_t group_size = KV_QUANT_GROUP_SIZE); - -inline size_t kv_scales_count(size_t seq_len, size_t kv_heads, size_t head_dim, size_t group_size = KV_QUANT_GROUP_SIZE) { - size_t num_groups = (head_dim + group_size - 1) / group_size; - return seq_len * kv_heads * num_groups; -} - -void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count); - -void cactus_gaussian_topk_f16( - const __fp16* input, - __fp16* output, - size_t rows, - size_t cols, - float ppf); - -void cactus_altup_predict_f16( - const __fp16* coefs, - const __fp16* const* streams, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim); - -void cactus_altup_correct_f16( - const __fp16* coefs, - const __fp16* innovation, - const __fp16* const* predictions, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim); - -void cactus_lstm_cell_f16( - const __fp16* x_input, - const __fp16* h_prev, - const __fp16* c_prev, - const __fp16* weight_ih, - const __fp16* weight_hh, - const __fp16* bias_ih, - const __fp16* bias_hh, - __fp16* h_new, - __fp16* c_new, - size_t batch_size, - size_t input_size, - size_t hidden_size -); - -void cactus_bilstm_sequence_f16( - const __fp16* input, - const __fp16* weight_ih_fwd, - const __fp16* weight_hh_fwd, - const __fp16* bias_ih_fwd, - const __fp16* bias_hh_fwd, - const __fp16* weight_ih_bwd, - const __fp16* weight_hh_bwd, - const __fp16* bias_ih_bwd, - const __fp16* bias_hh_bwd, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t input_size, - size_t hidden_size -); - -void cactus_maxpool1d_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t channels, - size_t input_length, - size_t kernel_size, - size_t stride -); - -#endif diff --git a/cactus/kernel/kernel_altup.cpp b/cactus/kernel/kernel_altup.cpp deleted file mode 100644 index f15bf9030..000000000 --- a/cactus/kernel/kernel_altup.cpp +++ /dev/null @@ -1,180 +0,0 @@ -#include "kernel.h" -#include -#include -#include -#include - -void cactus_gaussian_topk_f16( - const __fp16* input, - __fp16* output, - size_t rows, - size_t cols, - float ppf -) { - static constexpr float MAX_SAFE_ABS = 240.0f; - - for (size_t r = 0; r < rows; r++) { - const __fp16* row_in = input + r * cols; - __fp16* row_out = output + r * cols; - - // Pass 1: compute mean and max absolute deviation - float32x4_t sum_vec = vdupq_n_f32(0.0f); - size_t d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_low_f16(v))); - sum_vec = vaddq_f32(sum_vec, vcvt_f32_f16(vget_high_f16(v))); - } - float sum = vaddvq_f32(sum_vec); - for (; d < cols; d++) { - sum += static_cast(row_in[d]); - } - float mu = sum / static_cast(cols); - - float32x4_t mu_vec = vdupq_n_f32(mu); - float32x4_t max_abs_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec); - float32x4_t hi = vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec); - max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(lo)); - max_abs_vec = vmaxq_f32(max_abs_vec, vabsq_f32(hi)); - } - float max_abs = vmaxvq_f32(max_abs_vec); - for (; d < cols; d++) { - float diff = static_cast(row_in[d]) - mu; - float a = diff < 0.0f ? -diff : diff; - if (a > max_abs) max_abs = a; - } - - // Safe scaling: scale = max(max_abs / 240, 1) - float scale = max_abs * (1.0f / MAX_SAFE_ABS); - if (scale < 1.0f) scale = 1.0f; - float inv_scale = 1.0f / scale; - - // Pass 2: compute variance of scaled deviations - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t var_sum_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), mu_vec), inv_scale_vec); - float32x4_t hi = vmulq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), mu_vec), inv_scale_vec); - var_sum_vec = vfmaq_f32(var_sum_vec, lo, lo); - var_sum_vec = vfmaq_f32(var_sum_vec, hi, hi); - } - float var_sum = vaddvq_f32(var_sum_vec); - for (; d < cols; d++) { - float ds = (static_cast(row_in[d]) - mu) * inv_scale; - var_sum += ds * ds; - } - float variance = var_sum / static_cast(cols); - float sigma = sqrtf(variance) * scale; - - // cutoff = mu + ppf * sigma, output = relu(input - cutoff) - float cutoff = mu + ppf * sigma; - float32x4_t cutoff_vec = vdupq_n_f32(cutoff); - float32x4_t zero_vec = vdupq_n_f32(0.0f); - d = 0; - for (; d + 8 <= cols; d += 8) { - float16x8_t v = vld1q_f16(row_in + d); - float32x4_t lo = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_low_f16(v)), cutoff_vec), zero_vec); - float32x4_t hi = vmaxq_f32(vsubq_f32(vcvt_f32_f16(vget_high_f16(v)), cutoff_vec), zero_vec); - vst1q_f16(row_out + d, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); - } - for (; d < cols; d++) { - float val = static_cast(row_in[d]) - cutoff; - row_out[d] = static_cast<__fp16>(val > 0.0f ? val : 0.0f); - } - } -} - -void cactus_altup_predict_f16( - const __fp16* coefs, - const __fp16* const* streams, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim -) { - if (n > 8) throw std::runtime_error("cactus_altup_predict_f16 expects n <= 8"); - const size_t coef_stride = n * n; - - for (size_t i = 0; i < n; i++) { - for (size_t s = 0; s < seq_len; s++) { - const __fp16* coef_row = coefs + s * coef_stride + i * n; - __fp16* out_row = output + (i * seq_len + s) * hidden_dim; - const __fp16* src_i = streams[i] + s * hidden_dim; - - float c[8]; - for (size_t j = 0; j < n; j++) { - c[j] = static_cast(coef_row[j]); - } - - size_t d = 0; - for (; d + 8 <= hidden_dim; d += 8) { - float32x4_t acc_lo = vcvt_f32_f16(vget_low_f16(vld1q_f16(src_i + d))); - float32x4_t acc_hi = vcvt_f32_f16(vget_high_f16(vld1q_f16(src_i + d))); - - for (size_t j = 0; j < n; j++) { - float16x8_t sj = vld1q_f16(streams[j] + s * hidden_dim + d); - float32x4_t cj = vdupq_n_f32(c[j]); - acc_lo = vfmaq_f32(acc_lo, vcvt_f32_f16(vget_low_f16(sj)), cj); - acc_hi = vfmaq_f32(acc_hi, vcvt_f32_f16(vget_high_f16(sj)), cj); - } - - vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(acc_lo), vcvt_f16_f32(acc_hi))); - } - for (; d < hidden_dim; d++) { - float acc = static_cast(src_i[d]); - for (size_t j = 0; j < n; j++) { - acc += c[j] * static_cast(streams[j][s * hidden_dim + d]); - } - out_row[d] = static_cast<__fp16>(acc); - } - } - } -} - -void cactus_altup_correct_f16( - const __fp16* coefs, - const __fp16* innovation, - const __fp16* const* predictions, - __fp16* output, - size_t n, - size_t seq_len, - size_t hidden_dim -) { - for (size_t i = 0; i < n; i++) { - for (size_t s = 0; s < seq_len; s++) { - float ci = static_cast(coefs[s * n + i]); - float32x4_t ci_vec = vdupq_n_f32(ci); - - const __fp16* pred_row = predictions[i] + s * hidden_dim; - const __fp16* innov_row = innovation + s * hidden_dim; - __fp16* out_row = output + (i * seq_len + s) * hidden_dim; - - size_t d = 0; - for (; d + 8 <= hidden_dim; d += 8) { - float16x8_t pred = vld1q_f16(pred_row + d); - float16x8_t innov = vld1q_f16(innov_row + d); - - float32x4_t p_lo = vcvt_f32_f16(vget_low_f16(pred)); - float32x4_t p_hi = vcvt_f32_f16(vget_high_f16(pred)); - float32x4_t i_lo = vcvt_f32_f16(vget_low_f16(innov)); - float32x4_t i_hi = vcvt_f32_f16(vget_high_f16(innov)); - - float32x4_t out_lo = vfmaq_f32(p_lo, i_lo, ci_vec); - float32x4_t out_hi = vfmaq_f32(p_hi, i_hi, ci_vec); - - vst1q_f16(out_row + d, vcombine_f16(vcvt_f16_f32(out_lo), vcvt_f16_f32(out_hi))); - } - for (; d < hidden_dim; d++) { - out_row[d] = static_cast<__fp16>( - static_cast(pred_row[d]) + ci * static_cast(innov_row[d]) - ); - } - } - } -} diff --git a/cactus/kernel/kernel_attention.cpp b/cactus/kernel/kernel_attention.cpp deleted file mode 100644 index ad4dd13a4..000000000 --- a/cactus/kernel/kernel_attention.cpp +++ /dev/null @@ -1,1351 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -#ifdef __APPLE__ -static void cactus_attention_f16_accelerate( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - size_t v_head_dim, - float scale, - size_t position_offset, - bool is_causal -) { - constexpr size_t BLOCK_SIZE = 64; - - const size_t group_size = num_q_heads / num_kv_heads; - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t k_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t k_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - static constexpr CactusThreading::ParallelConfig ATTENTION_BATCHED{1, 1}; - CactusThreading::parallel_for(batch_size * num_q_heads, ATTENTION_BATCHED, - [&](size_t start, size_t end) { - - std::vector Q_f32(seq_len * head_dim); - std::vector K_f32(BLOCK_SIZE * head_dim); - std::vector V_f32(BLOCK_SIZE * v_head_dim); - std::vector scores(seq_len * BLOCK_SIZE); - std::vector acc(seq_len * v_head_dim); - std::vector row_max(seq_len); - std::vector row_sum(seq_len); - - for (size_t work = start; work < end; ++work) { - const size_t batch = work / num_q_heads; - const size_t q_head = work % num_q_heads; - const size_t kv_head = q_head / group_size; - - for (size_t q = 0; q < seq_len; ++q) { - const __fp16* q_src = queries + batch*q_batch_stride + q*q_seq_stride + q_head*head_dim; - float* q_dst = Q_f32.data() + q * head_dim; - for (size_t d = 0; d < head_dim; d += 8) { - float16x8_t v = vld1q_f16(q_src + d); - vst1q_f32(q_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(q_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - std::fill(row_max.begin(), row_max.begin() + seq_len, -INFINITY); - std::fill(row_sum.begin(), row_sum.begin() + seq_len, 0.0f); - memset(acc.data(), 0, seq_len * v_head_dim * sizeof(float)); - - for (size_t kv0 = 0; kv0 < kv_seq_len; kv0 += BLOCK_SIZE) { - const size_t block_len = std::min(BLOCK_SIZE, kv_seq_len - kv0); - - size_t q_start = 0; - size_t active_rows = seq_len; - if (is_causal) { - if (kv0 > position_offset) { - q_start = kv0 - position_offset; - } - if (q_start >= seq_len) continue; - active_rows = seq_len - q_start; - } - - for (size_t i = 0; i < block_len; ++i) { - const __fp16* k_src = keys + batch*k_batch_stride + (kv0+i)*k_seq_stride + kv_head*head_dim; - float* k_dst = K_f32.data() + i * head_dim; - for (size_t d = 0; d < head_dim; d += 8) { - float16x8_t v = vld1q_f16(k_src + d); - vst1q_f32(k_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(k_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - (int)active_rows, (int)block_len, (int)head_dim, - scale, - Q_f32.data() + q_start * head_dim, (int)head_dim, - K_f32.data(), (int)head_dim, - 0.0f, - scores.data(), (int)block_len); - - for (size_t r = 0; r < active_rows; ++r) { - const size_t q_pos = q_start + r; - const size_t abs_q = position_offset + q_pos; - float* s_row = scores.data() + r * block_len; - - size_t valid_len = block_len; - if (is_causal) { - if (abs_q < kv0) { - memset(s_row, 0, block_len * sizeof(float)); - continue; - } - valid_len = std::min(block_len, abs_q - kv0 + 1); - } - - float32x4_t vmax = vdupq_n_f32(-INFINITY); - size_t j = 0; - for (; j + 4 <= valid_len; j += 4) { - vmax = vmaxq_f32(vmax, vld1q_f32(s_row + j)); - } - float block_max = vmaxvq_f32(vmax); - for (; j < valid_len; ++j) { - block_max = std::max(block_max, s_row[j]); - } - - float prev_max = row_max[q_pos]; - float new_max = std::max(prev_max, block_max); - float scale_old = expf(prev_max - new_max); - - if (prev_max != -INFINITY) { - float* acc_row = acc.data() + q_pos * v_head_dim; - float32x4_t sv = vdupq_n_f32(scale_old); - for (size_t d = 0; d < v_head_dim; d += 4) { - float32x4_t a = vld1q_f32(acc_row + d); - vst1q_f32(acc_row + d, vmulq_f32(a, sv)); - } - } - row_sum[q_pos] = row_sum[q_pos] * scale_old; - row_max[q_pos] = new_max; - - float32x4_t vsum = vdupq_n_f32(0.0f); - float32x4_t vnmax = vdupq_n_f32(new_max); - float32x4_t log2e = vdupq_n_f32(1.442695f); - j = 0; - for (; j + 4 <= valid_len; j += 4) { - float32x4_t x = vmulq_f32(vsubq_f32(vld1q_f32(s_row + j), vnmax), log2e); - float32x4_t x_floor = vrndmq_f32(x); - int32x4_t xi = vcvtq_s32_f32(x_floor); - float32x4_t xf = vsubq_f32(x, x_floor); - float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); - t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); - float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); - xi = vshlq_n_s32(vaddq_s32(xi, vdupq_n_s32(127)), 23); - y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); - uint32x4_t underflow = vcltq_f32(x, vdupq_n_f32(-126.0f)); - y = vbslq_f32(underflow, vdupq_n_f32(0.0f), y); - vst1q_f32(s_row + j, y); - vsum = vaddq_f32(vsum, y); - } - float block_sum = vaddvq_f32(vsum); - for (; j < valid_len; ++j) { - s_row[j] = expf(s_row[j] - new_max); - block_sum += s_row[j]; - } - if (valid_len < block_len) { - memset(s_row + valid_len, 0, (block_len - valid_len) * sizeof(float)); - } - row_sum[q_pos] += block_sum; - } - - for (size_t i = 0; i < block_len; ++i) { - const __fp16* v_src = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; - float* v_dst = V_f32.data() + i * v_head_dim; - for (size_t d = 0; d < v_head_dim; d += 8) { - float16x8_t v = vld1q_f16(v_src + d); - vst1q_f32(v_dst + d, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(v_dst + d + 4, vcvt_f32_f16(vget_high_f16(v))); - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - (int)active_rows, (int)v_head_dim, (int)block_len, - 1.0f, - scores.data(), (int)block_len, - V_f32.data(), (int)v_head_dim, - 1.0f, - acc.data() + q_start * v_head_dim, (int)v_head_dim); - } - - for (size_t q = 0; q < seq_len; ++q) { - __fp16* o = output + batch*o_batch_stride + q*o_seq_stride + q_head*v_head_dim; - float sum = row_sum[q]; - if (sum == 0.0f) { - memset(o, 0, v_head_dim * sizeof(__fp16)); - continue; - } - float inv = 1.0f / sum; - float32x4_t invv = vdupq_n_f32(inv); - float* acc_row = acc.data() + q * v_head_dim; - for (size_t d = 0; d < v_head_dim; d += 8) { - float32x4_t a0 = vmulq_f32(vld1q_f32(acc_row + d), invv); - float32x4_t a1 = vmulq_f32(vld1q_f32(acc_row + d + 4), invv); - vst1q_f16(o + d, vcombine_f16(vcvt_f16_f32(a0), vcvt_f16_f32(a1))); - } - } - } - }); -} -#endif - -static inline void cactus_attention_f16_fast( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - size_t position_offset, - bool is_causal, - size_t window_size, - size_t v_head_dim -) { - constexpr size_t BLOCK_SIZE = 32; - const size_t qk_nblocks = head_dim / 8; - const size_t v_nblocks = v_head_dim / 8; - -#ifdef __APPLE__ - if (seq_len >= 64 && window_size == 0) { - cactus_attention_f16_accelerate( - queries, keys, values, output, - batch_size, seq_len, kv_seq_len, - num_q_heads, num_kv_heads, head_dim, v_head_dim, - scale, position_offset, is_causal - ); - return; - } -#endif - - const size_t group_size = num_q_heads / num_kv_heads; - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t kv_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [&](size_t start, size_t end) { - - float block_scores[BLOCK_SIZE]; - std::vector acc_lo(v_nblocks), acc_hi(v_nblocks); - - for (size_t work = start; work < end; ++work) { - const size_t batch = work / (num_q_heads * seq_len); - const size_t rem = work % (num_q_heads * seq_len); - const size_t q_head = rem / seq_len; - const size_t q_pos = rem % seq_len; - const size_t kv_head = q_head / group_size; - - const __fp16* q = queries + batch*q_batch_stride + q_pos*q_seq_stride + q_head*head_dim; - __fp16* o = output + batch*o_batch_stride + q_pos*o_seq_stride + q_head*v_head_dim; - - for (size_t i = 0; i < v_nblocks; i++) { - acc_lo[i] = vdupq_n_f32(0.f); - acc_hi[i] = vdupq_n_f32(0.f); - } - - float running_max = -INFINITY; - float running_sum = 0.f; - - const size_t abs_q = position_offset + q_pos; - size_t kv_end = is_causal ? std::min(kv_seq_len, abs_q + 1) : kv_seq_len; - size_t kv_start = (window_size > 0 && abs_q > window_size) ? abs_q - window_size : 0; - - for (size_t kv0 = kv_start; kv0 < kv_end; kv0 += BLOCK_SIZE) { - const size_t kv1 = std::min(kv0 + BLOCK_SIZE, kv_end); - float block_max = -INFINITY; - - for (size_t i = kv0; i < kv1; i++) { - float32x4_t s0 = vdupq_n_f32(0.f); - float32x4_t s1 = vdupq_n_f32(0.f); - - const __fp16* k = keys + batch*kv_batch_stride + i*kv_seq_stride + kv_head*head_dim; - - for (size_t d = 0; d < qk_nblocks; d++) { - float16x8_t qv = vld1q_f16(q + d*8); - float16x8_t kv = vld1q_f16(k + d*8); - - float32x4_t ql = vcvt_f32_f16(vget_low_f16(qv)); - float32x4_t qh = vcvt_f32_f16(vget_high_f16(qv)); - float32x4_t kl = vcvt_f32_f16(vget_low_f16(kv)); - float32x4_t kh = vcvt_f32_f16(vget_high_f16(kv)); - - s0 = vfmaq_f32(s0, ql, kl); - s1 = vfmaq_f32(s1, qh, kh); - } - - float score = vaddvq_f32(vaddq_f32(s0, s1)) * scale; - block_scores[i - kv0] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t d = 0; d < v_nblocks; d++) { - acc_lo[d] = vmulq_n_f32(acc_lo[d], scale_correction); - acc_hi[d] = vmulq_n_f32(acc_hi[d], scale_correction); - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - - float block_sum = 0.f; - for (size_t i = 0; i < kv1 - kv0; i++) { - block_scores[i] = expf(block_scores[i] - block_max); - block_sum += block_scores[i]; - } - - for (size_t i = 0; i < kv1 - kv0; i++) { - const float attn_weight = block_scores[i] * current_block_scale; - if (attn_weight == 0.f) continue; - - const __fp16* v = values + batch*v_batch_stride + (kv0+i)*v_seq_stride + kv_head*v_head_dim; - float32x4_t wv = vdupq_n_f32(attn_weight); - - for (size_t d = 0; d < v_nblocks; d++) { - float16x8_t vv = vld1q_f16(v + d*8); - acc_lo[d] = vfmaq_f32(acc_lo[d], vcvt_f32_f16(vget_low_f16(vv)), wv); - acc_hi[d] = vfmaq_f32(acc_hi[d], vcvt_f32_f16(vget_high_f16(vv)), wv); - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum == 0.f) { - memset(o, 0, v_head_dim * sizeof(__fp16)); - continue; - } - - float inv = 1.f / running_sum; - float32x4_t invv = vdupq_n_f32(inv); - - for (size_t d = 0; d < v_nblocks; d++) { - float16x8_t out = vcombine_f16( - vcvt_f16_f32(vmulq_f32(acc_lo[d], invv)), - vcvt_f16_f32(vmulq_f32(acc_hi[d], invv)) - ); - vst1q_f16(o + d*8, out); - } - } - }); -} - -void cactus_attention_f16( - const __fp16* queries, - const __fp16* keys, - const __fp16* values, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t kv_seq_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - const __fp16* mask, - size_t position_offset, - size_t window_size, - bool is_causal, - bool mask_is_additive, - bool mask_per_head, - size_t v_head_dim, - float logit_cap -) { - if (v_head_dim == 0) v_head_dim = head_dim; - if (scale == 0.0f) { - scale = 1.0f / sqrtf(static_cast(head_dim)); - } - - if (mask == nullptr && head_dim % 8 == 0 && v_head_dim % 8 == 0 && logit_cap == 0.0f) { - cactus_attention_f16_fast( - queries, keys, values, output, - batch_size, seq_len, kv_seq_len, - num_q_heads, num_kv_heads, head_dim, - scale, position_offset, is_causal, window_size, v_head_dim - ); - return; - } - - constexpr size_t VECTOR_WIDTH = 8; - constexpr size_t BLOCK_SIZE = 32; - const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - - const size_t group_size = num_q_heads / num_kv_heads; - - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t kv_batch_stride = kv_seq_len * num_kv_heads * head_dim; - const size_t v_batch_stride = kv_seq_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t kv_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - const size_t mask_batch_stride = mask - ? (mask_per_head ? (num_q_heads * seq_len * kv_seq_len) : (seq_len * kv_seq_len)) - : 0; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [=](size_t start_idx, size_t end_idx) { - std::vector block_scores(BLOCK_SIZE); - std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); - std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); - - const size_t v_tail_dims = v_head_dim - v_head_dim_aligned; - std::vector output_accum_tail(v_tail_dims, 0.0f); - - const float NEG_INF = -std::numeric_limits::infinity(); - const size_t used_vec_blocks = v_head_dim_aligned / VECTOR_WIDTH; - - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t batch_idx = work_idx / (num_q_heads * seq_len); - const size_t remainder = work_idx % (num_q_heads * seq_len); - const size_t q_head_idx = remainder / seq_len; - const size_t q_pos = remainder % seq_len; - - const size_t kv_head_idx = q_head_idx / group_size; - - const __fp16* Q_base = queries + batch_idx * q_batch_stride; - const __fp16* K_base = keys + batch_idx * kv_batch_stride; - const __fp16* V_base = values + batch_idx * v_batch_stride; - __fp16* O_base = output + batch_idx * o_batch_stride; - const __fp16* M = mask ? (mask + batch_idx * mask_batch_stride) : nullptr; - const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; - __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; - - float running_max = -std::numeric_limits::infinity(); - float running_sum = 0.0f; - - for (size_t i = 0; i < output_accum_low.size(); ++i) { - output_accum_low[i] = vdupq_n_f32(0.0f); - output_accum_high[i] = vdupq_n_f32(0.0f); - } - for (size_t i = 0; i < v_tail_dims; ++i) { - output_accum_tail[i] = 0.0f; - } - - const bool is_decode = (q_pos == seq_len - 1) && seq_len > 1; - const size_t absolute_q_pos = position_offset + q_pos; - - size_t kv_start = 0; - size_t kv_end = kv_seq_len; - - if (window_size > 0 && window_size < kv_seq_len) { - if (absolute_q_pos > window_size) { - kv_start = absolute_q_pos - window_size; - } - if (is_causal) { - kv_end = std::min(kv_end, absolute_q_pos + 1); - } - } else if (is_causal) { - kv_end = std::min(kv_end, absolute_q_pos + 1); - } - - for (size_t kv_block_start = kv_start; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { - const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); - const size_t block_size = kv_block_end - kv_block_start; - - float block_max = -std::numeric_limits::infinity(); - - if (!is_decode && is_causal && kv_block_start > absolute_q_pos) { - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - block_scores[kv_idx] = NEG_INF; - } - continue; - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const size_t kv_pos = kv_block_start + kv_idx; - - if (!is_decode && is_causal && kv_pos > absolute_q_pos) { - block_scores[kv_idx] = NEG_INF; - continue; - } - - const __fp16* k_vec = K_base + kv_pos * kv_seq_stride + kv_head_idx * head_dim; - - if (kv_idx + 1 < block_size) { - const __fp16* next_k_vec = K_base + (kv_pos + 1) * kv_seq_stride + kv_head_idx * head_dim; - __builtin_prefetch(next_k_vec, 0, 1); - } - - float32x4_t score_accum_low = vdupq_n_f32(0.0f); - float32x4_t score_accum_high = vdupq_n_f32(0.0f); - - for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); - - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); - float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - - float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)); - - for (size_t dim = head_dim_aligned; dim < head_dim; ++dim) { - score += static_cast(q_vec[dim]) * static_cast(k_vec[dim]); - } - - score *= scale; - - size_t absolute_q_pos = position_offset + q_pos; - - if (is_causal && kv_pos > absolute_q_pos) { - score = NEG_INF; - } - else if (window_size > 0 && kv_pos < absolute_q_pos && (absolute_q_pos - kv_pos) > window_size) { - score = NEG_INF; - } - else if (M) { - const size_t mask_index = mask_per_head - ? ((q_head_idx * seq_len + q_pos) * kv_seq_len + kv_pos) - : (q_pos * kv_seq_len + kv_pos); - const float mask_value = static_cast(M[mask_index]); - if (mask_is_additive) { - if (!std::isfinite(mask_value)) { - score = NEG_INF; - } else { - score += mask_value; - } - } else if (mask_value == 0.0f) { - score = NEG_INF; - } - } - - if (logit_cap > 0.0f && std::isfinite(score)) { - score = logit_cap * tanhf(score / logit_cap); - } - - block_scores[kv_idx] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - - if (block_max > NEG_INF) { - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t i = 0; i < used_vec_blocks; ++i) { - output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); - output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); - } - for (size_t i = 0; i < v_tail_dims; ++i) { - output_accum_tail[i] *= scale_correction; - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - } - - float block_sum = 0.0f; - const size_t vec_size = (block_size / 4) * 4; - - for (size_t kv_idx = 0; kv_idx < vec_size; kv_idx += 4) { - float32x4_t scores = vld1q_f32(&block_scores[kv_idx]); - uint32x4_t inf_mask = vceqq_f32(scores, vdupq_n_f32(NEG_INF)); - - float32x4_t x = vsubq_f32(scores, vdupq_n_f32(block_max)); - x = vmulq_n_f32(x, 1.442695f); - float32x4_t x_floor = vrndmq_f32(x); - int32x4_t xi = vcvtq_s32_f32(x_floor); - float32x4_t xf = vsubq_f32(x, x_floor); - - float32x4_t t = vfmaq_n_f32(vdupq_n_f32(0.2246932f), xf, 0.0789673f); - t = vfmaq_f32(vdupq_n_f32(0.6963248f), t, xf); - float32x4_t y = vfmaq_f32(vdupq_n_f32(0.9999003f), t, xf); - - xi = vaddq_s32(xi, vdupq_n_s32(127)); - xi = vshlq_n_s32(xi, 23); - y = vmulq_f32(y, vreinterpretq_f32_s32(xi)); - - uint32x4_t underflow_mask = vcltq_f32(x, vdupq_n_f32(-126.0f)); - uint32x4_t zero_mask = vorrq_u32(inf_mask, underflow_mask); - y = vbslq_f32(zero_mask, vdupq_n_f32(0.0f), y); - - vst1q_f32(&block_scores[kv_idx], y); - block_sum += vaddvq_f32(y); - } - - for (size_t kv_idx = vec_size; kv_idx < block_size; ++kv_idx) { - if (block_scores[kv_idx] != NEG_INF) { - block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); - block_sum += block_scores[kv_idx]; - } else { - block_scores[kv_idx] = 0.0f; - } - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const float attn_weight = block_scores[kv_idx] * current_block_scale; - if (attn_weight == 0.0f) continue; - - const size_t kv_pos = kv_block_start + kv_idx; - const __fp16* v_vec = V_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; - - const float32x4_t weight_vec = vdupq_n_f32(attn_weight); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); - float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); - float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - - for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { - float val = attn_weight * static_cast(v_vec[dim]); - output_accum_tail[dim - v_head_dim_aligned] += val; - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum > 0.0f) { - const float inv_sum = 1.0f / running_sum; - const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - size_t idx = dim_block / VECTOR_WIDTH; - float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); - float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); - - float16x4_t low_f16 = vcvt_f16_f32(final_low); - float16x4_t high_f16 = vcvt_f16_f32(final_high); - float16x8_t combined = vcombine_f16(low_f16, high_f16); - - vst1q_f16(&o_vec[dim_block], combined); - } - - for (size_t dim = v_head_dim_aligned; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(output_accum_tail[dim - v_head_dim_aligned] * inv_sum); - } - } else { - for (size_t dim = 0; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(0.0f); - } - } - } - }); -} - -void cactus_attention_hybrid_int8_fp16( - const __fp16* queries, - const int8_t* keys_cached, - const int8_t* values_cached, - const float* k_scales, - const float* v_scales, - const __fp16* keys_new, - const __fp16* values_new, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t cache_len, - size_t new_len, - size_t num_q_heads, - size_t num_kv_heads, - size_t head_dim, - float scale, - size_t position_offset, - bool is_causal, - size_t window_size, - size_t quant_group_size, - size_t v_head_dim -) { - if (v_head_dim == 0) v_head_dim = head_dim; - if (scale == 0.0f) { - scale = 1.0f / sqrtf(static_cast(head_dim)); - } - - const size_t kv_seq_len = cache_len + new_len; - - constexpr size_t VECTOR_WIDTH = 8; - constexpr size_t BLOCK_SIZE = 32; - const size_t head_dim_aligned = (head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - const size_t v_head_dim_aligned = (v_head_dim / VECTOR_WIDTH) * VECTOR_WIDTH; - - const size_t gqa_group_size = num_q_heads / num_kv_heads; - const size_t num_quant_groups_k = (head_dim + quant_group_size - 1) / quant_group_size; - const size_t num_quant_groups_v = (v_head_dim + quant_group_size - 1) / quant_group_size; - - const size_t q_batch_stride = seq_len * num_q_heads * head_dim; - const size_t k_cached_batch_stride = cache_len * num_kv_heads * head_dim; - const size_t v_cached_batch_stride = cache_len * num_kv_heads * v_head_dim; - const size_t k_new_batch_stride = new_len * num_kv_heads * head_dim; - const size_t v_new_batch_stride = new_len * num_kv_heads * v_head_dim; - const size_t o_batch_stride = seq_len * num_q_heads * v_head_dim; - const size_t q_seq_stride = num_q_heads * head_dim; - const size_t k_seq_stride = num_kv_heads * head_dim; - const size_t v_seq_stride = num_kv_heads * v_head_dim; - const size_t o_seq_stride = num_q_heads * v_head_dim; - - CactusThreading::parallel_for(batch_size * num_q_heads * seq_len, CactusThreading::Thresholds::ATTENTION, - [=](size_t start_idx, size_t end_idx) { - std::vector block_scores(BLOCK_SIZE); - std::vector output_accum_low(v_head_dim_aligned / VECTOR_WIDTH * 2); - std::vector output_accum_high(v_head_dim_aligned / VECTOR_WIDTH * 2); - - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t batch_idx = work_idx / (num_q_heads * seq_len); - const size_t remainder = work_idx % (num_q_heads * seq_len); - const size_t q_head_idx = remainder / seq_len; - const size_t q_pos = remainder % seq_len; - - const size_t kv_head_idx = q_head_idx / gqa_group_size; - - const __fp16* Q_base = queries + batch_idx * q_batch_stride; - const int8_t* K_cached_base = keys_cached + batch_idx * k_cached_batch_stride; - const int8_t* V_cached_base = values_cached + batch_idx * v_cached_batch_stride; - const __fp16* K_new_base = keys_new + batch_idx * k_new_batch_stride; - const __fp16* V_new_base = values_new + batch_idx * v_new_batch_stride; - __fp16* O_base = output + batch_idx * o_batch_stride; - - const __fp16* q_vec = Q_base + q_pos * q_seq_stride + q_head_idx * head_dim; - __fp16* o_vec = O_base + q_pos * o_seq_stride + q_head_idx * v_head_dim; - - float running_max = -std::numeric_limits::infinity(); - float running_sum = 0.0f; - - for (size_t i = 0; i < output_accum_low.size(); ++i) { - output_accum_low[i] = vdupq_n_f32(0.0f); - output_accum_high[i] = vdupq_n_f32(0.0f); - } - - const size_t absolute_q_pos = position_offset + q_pos; - size_t kv_end = is_causal ? std::min(kv_seq_len, cache_len + q_pos + 1) : kv_seq_len; - - size_t kv_start = 0; - if (window_size > 0 && absolute_q_pos > window_size) { - kv_start = absolute_q_pos - window_size; - } - - constexpr size_t SINK_SIZE = 4; - const size_t cache_abs_offset = (position_offset >= cache_len) ? (position_offset - cache_len) : 0; - - const size_t kv_block_start0 = (window_size > 0 && kv_start > 0) ? 0 - : (kv_start / BLOCK_SIZE) * BLOCK_SIZE; - - for (size_t kv_block_start = kv_block_start0; kv_block_start < kv_end; kv_block_start += BLOCK_SIZE) { - const size_t kv_block_end = std::min(kv_block_start + BLOCK_SIZE, kv_end); - const size_t block_size = kv_block_end - kv_block_start; - - float block_max = -std::numeric_limits::infinity(); - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const size_t kv_pos = kv_block_start + kv_idx; - - bool window_masked = false; - if (window_size > 0 && kv_start > 0) { - if (kv_pos < cache_len) { - if (cache_abs_offset == 0 || kv_pos >= SINK_SIZE) { - window_masked = (cache_abs_offset + kv_pos < kv_start); - } - } else { - window_masked = (kv_pos + cache_abs_offset < kv_start); - } - } - - if ((is_causal && kv_pos > absolute_q_pos) || window_masked) { - block_scores[kv_idx] = -std::numeric_limits::infinity(); - continue; - } - - float32x4_t score_accum_low = vdupq_n_f32(0.0f); - float32x4_t score_accum_high = vdupq_n_f32(0.0f); - - if (kv_pos < cache_len) { - const int8_t* k_vec = K_cached_base + kv_pos * k_seq_stride + kv_head_idx * head_dim; - const float* k_scale_base = k_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_k; - - for (size_t quant_group = 0; quant_group < num_quant_groups_k; quant_group++) { - const size_t dim_base = quant_group * quant_group_size; - const float k_scale = k_scale_base[quant_group]; - const float32x4_t k_scale_vec = vdupq_n_f32(k_scale); - - #pragma unroll - for (size_t i = 0; i < 4; i++) { - const size_t dim_block = dim_base + i * VECTOR_WIDTH; - if (dim_block >= head_dim_aligned) break; - - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - - int8x8_t k_vec_i8 = vld1_s8(&k_vec[dim_block]); - int16x8_t k_vec_i16 = vmovl_s8(k_vec_i8); - float32x4_t k_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(k_vec_i16))), k_scale_vec); - float32x4_t k_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(k_vec_i16))), k_scale_vec); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - } - } else { - const size_t new_pos = kv_pos - cache_len; - const __fp16* k_vec = K_new_base + new_pos * k_seq_stride + kv_head_idx * head_dim; - - for (size_t dim_block = 0; dim_block < head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t q_vec_f16 = vld1q_f16(&q_vec[dim_block]); - float16x8_t k_vec_f16 = vld1q_f16(&k_vec[dim_block]); - - float32x4_t q_low = vcvt_f32_f16(vget_low_f16(q_vec_f16)); - float32x4_t q_high = vcvt_f32_f16(vget_high_f16(q_vec_f16)); - float32x4_t k_low = vcvt_f32_f16(vget_low_f16(k_vec_f16)); - float32x4_t k_high = vcvt_f32_f16(vget_high_f16(k_vec_f16)); - - score_accum_low = vfmaq_f32(score_accum_low, q_low, k_low); - score_accum_high = vfmaq_f32(score_accum_high, q_high, k_high); - } - } - - float score = vaddvq_f32(vaddq_f32(score_accum_low, score_accum_high)) * scale; - block_scores[kv_idx] = score; - block_max = std::max(block_max, score); - } - - float current_block_scale = 1.0f; - if (block_max > -std::numeric_limits::infinity()) { - if (block_max > running_max) { - float scale_correction = expf(running_max - block_max); - running_sum *= scale_correction; - - for (size_t i = 0; i < output_accum_low.size() / 2; ++i) { - output_accum_low[i] = vmulq_n_f32(output_accum_low[i], scale_correction); - output_accum_high[i] = vmulq_n_f32(output_accum_high[i], scale_correction); - } - running_max = block_max; - } else { - current_block_scale = expf(block_max - running_max); - } - } - - float block_sum = 0.0f; - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - if (block_scores[kv_idx] != -std::numeric_limits::infinity()) { - block_scores[kv_idx] = expf(block_scores[kv_idx] - block_max); - block_sum += block_scores[kv_idx]; - } else { - block_scores[kv_idx] = 0.0f; - } - } - - for (size_t kv_idx = 0; kv_idx < block_size; ++kv_idx) { - const float attn_weight = block_scores[kv_idx] * current_block_scale; - if (attn_weight == 0.0f) continue; - - const size_t kv_pos = kv_block_start + kv_idx; - const float32x4_t weight_vec = vdupq_n_f32(attn_weight); - - if (kv_pos < cache_len) { - const int8_t* v_vec = V_cached_base + kv_pos * v_seq_stride + kv_head_idx * v_head_dim; - const float* v_scale_base = v_scales + (kv_pos * num_kv_heads + kv_head_idx) * num_quant_groups_v; - - for (size_t quant_group = 0; quant_group < num_quant_groups_v; quant_group++) { - const size_t dim_base = quant_group * quant_group_size; - const float v_scale = v_scale_base[quant_group]; - const float32x4_t v_scale_vec = vdupq_n_f32(v_scale); - - #pragma unroll - for (size_t i = 0; i < 4; i++) { - const size_t dim_block = dim_base + i * VECTOR_WIDTH; - if (dim_block >= v_head_dim_aligned) break; - - int8x8_t v_vec_i8 = vld1_s8(&v_vec[dim_block]); - int16x8_t v_vec_i16 = vmovl_s8(v_vec_i8); - float32x4_t v_low = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(v_vec_i16))), v_scale_vec); - float32x4_t v_high = vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(v_vec_i16))), v_scale_vec); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - } - } else { - const size_t new_pos = kv_pos - cache_len; - const __fp16* v_vec = V_new_base + new_pos * v_seq_stride + kv_head_idx * v_head_dim; - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - float16x8_t v_vec_f16 = vld1q_f16(&v_vec[dim_block]); - float32x4_t v_low = vcvt_f32_f16(vget_low_f16(v_vec_f16)); - float32x4_t v_high = vcvt_f32_f16(vget_high_f16(v_vec_f16)); - - size_t idx = dim_block / VECTOR_WIDTH; - output_accum_low[idx] = vfmaq_f32(output_accum_low[idx], v_low, weight_vec); - output_accum_high[idx] = vfmaq_f32(output_accum_high[idx], v_high, weight_vec); - } - } - } - - running_sum += block_sum * current_block_scale; - } - - if (running_sum > 0.0f) { - const float inv_sum = 1.0f / running_sum; - const float32x4_t inv_sum_vec = vdupq_n_f32(inv_sum); - - for (size_t dim_block = 0; dim_block < v_head_dim_aligned; dim_block += VECTOR_WIDTH) { - size_t idx = dim_block / VECTOR_WIDTH; - float32x4_t final_low = vmulq_f32(output_accum_low[idx], inv_sum_vec); - float32x4_t final_high = vmulq_f32(output_accum_high[idx], inv_sum_vec); - - float16x4_t low_f16 = vcvt_f16_f32(final_low); - float16x4_t high_f16 = vcvt_f16_f32(final_high); - float16x8_t combined = vcombine_f16(low_f16, high_f16); - - vst1q_f16(&o_vec[dim_block], combined); - } - } else { - for (size_t dim = 0; dim < v_head_dim; ++dim) { - o_vec[dim] = static_cast<__fp16>(0.0f); - } - } - } - }); -} - -void cactus_rms_norm_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t batch_size, - size_t dims, - float eps -) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 2; - constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* input_row = input + b * dims; - __fp16* output_row = output + b * dims; - - float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_squares_vec[u] = vdupq_n_f32(0.0f); - } - - size_t i = 0; - const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; - - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); - sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); - } - } - - const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); - sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); - } - - float32x4_t total_sum = sum_squares_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - total_sum = vaddq_f32(total_sum, sum_squares_vec[u]); - } - float sum_squares = vaddvq_f32(total_sum); - - for (; i < dims; ++i) { - float val = static_cast(input_row[i]); - sum_squares += val * val; - } - - float rms = sqrtf(sum_squares / static_cast(dims) + eps); - float inv_rms = 1.0f / rms; - float16x8_t inv_rms_vec = vdupq_n_f16(static_cast<__fp16>(inv_rms)); - - i = 0; - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], norm_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t norm_vec = vmulq_f16(vmulq_f16(input_vec, inv_rms_vec), weight_vec); - vst1q_f16(&output_row[i], norm_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>(static_cast(input_row[i]) * inv_rms * static_cast(weight[i])); - } - } -} - -void cactus_layer_norm_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t batch_size, - size_t dims, - float eps -) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 3; - constexpr size_t TILE_SIZE = SIMD_WIDTH * UNROLL_FACTOR; - - const size_t tile_end = (dims >= TILE_SIZE) ? dims - TILE_SIZE + 1 : 0; - const size_t simd_end = (dims >= SIMD_WIDTH) ? dims - SIMD_WIDTH + 1 : 0; - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* input_row = input + b * dims; - __fp16* output_row = output + b * dims; - - float32x4_t sum_input_vec[UNROLL_FACTOR * 2]; - float32x4_t sum_squares_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_input_vec[u] = vdupq_n_f32(0.0f); - sum_squares_vec[u] = vdupq_n_f32(0.0f); - } - - size_t i = 0; - - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - - sum_input_vec[u * 2] = vaddq_f32(sum_input_vec[u * 2], input_low); - sum_input_vec[u * 2 + 1] = vaddq_f32(sum_input_vec[u * 2 + 1], input_high); - - sum_squares_vec[u * 2] = vfmaq_f32(sum_squares_vec[u * 2], input_low, input_low); - sum_squares_vec[u * 2 + 1] = vfmaq_f32(sum_squares_vec[u * 2 + 1], input_high, input_high); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input_vec)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input_vec)); - sum_input_vec[0] = vaddq_f32(sum_input_vec[0], input_low); - sum_input_vec[1] = vaddq_f32(sum_input_vec[1], input_high); - sum_squares_vec[0] = vfmaq_f32(sum_squares_vec[0], input_low, input_low); - sum_squares_vec[1] = vfmaq_f32(sum_squares_vec[1], input_high, input_high); - } - - float32x4_t total_sum_inputs = sum_input_vec[0]; - float32x4_t total_sum_squares = sum_squares_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - total_sum_inputs = vaddq_f32(total_sum_inputs, sum_input_vec[u]); - total_sum_squares = vaddq_f32(total_sum_squares, sum_squares_vec[u]); - } - - float sum_inputs = vaddvq_f32(total_sum_inputs); - float sum_squares = vaddvq_f32(total_sum_squares); - for (; i < dims; ++i) { - float val = static_cast(input_row[i]); - sum_inputs += val; - sum_squares += val * val; - } - - float mean = sum_inputs / static_cast(dims); - float mean_squares = sum_squares / static_cast(dims); - float variance = mean_squares - mean * mean; - if (variance < 0.0f) variance = 0.0f; - float inv_std = 1.0f / sqrtf(variance + eps); - - float16x8_t mean_vec = vdupq_n_f16(static_cast<__fp16>(mean)); - float16x8_t inv_std_vec = vdupq_n_f16(static_cast<__fp16>(inv_std)); - - i = 0; - if (bias) { - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t bias_vec = vld1q_f16(&bias[i + u * SIMD_WIDTH]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - out_vec = vaddq_f16(out_vec, bias_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t bias_vec = vld1q_f16(&bias[i]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - out_vec = vaddq_f16(out_vec, bias_vec); - vst1q_f16(&output_row[i], out_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i]) + static_cast(bias[i])); - } - } else { - for (; i < tile_end; i += TILE_SIZE) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t input_vec = vld1q_f16(&input_row[i + u * SIMD_WIDTH]); - float16x8_t weight_vec = vld1q_f16(&weight[i + u * SIMD_WIDTH]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - vst1q_f16(&output_row[i + u * SIMD_WIDTH], out_vec); - } - } - - for (; i < simd_end; i += SIMD_WIDTH) { - float16x8_t input_vec = vld1q_f16(&input_row[i]); - float16x8_t weight_vec = vld1q_f16(&weight[i]); - float16x8_t out_vec = vmulq_f16(vmulq_f16(vsubq_f16(input_vec, mean_vec), inv_std_vec), weight_vec); - vst1q_f16(&output_row[i], out_vec); - } - - for (; i < dims; ++i) { - output_row[i] = static_cast<__fp16>((static_cast(input_row[i]) - mean) * inv_std * static_cast(weight[i])); - } - } - } -} - -namespace CactusRoPEF16 { - -struct RoPECacheF16 { - std::vector<__fp16> cos_table; - std::vector<__fp16> sin_table; - size_t max_seq_len; - size_t head_dim; - float theta; - bool initialized; - - RoPECacheF16() : max_seq_len(0), head_dim(0), theta(0.0f), initialized(false) {} -}; - -static thread_local RoPECacheF16 rope_cache_f16; - -void precompute_rope_tables_f16(size_t seq_len, size_t head_dim, float theta) { - if (rope_cache_f16.initialized && - rope_cache_f16.max_seq_len >= seq_len && - rope_cache_f16.head_dim == head_dim && - rope_cache_f16.theta == theta) { - return; - } - - const size_t half_dim = head_dim / 2; - const size_t table_size = seq_len * half_dim; - - rope_cache_f16.cos_table.resize(table_size); - rope_cache_f16.sin_table.resize(table_size); - - for (size_t pos = 0; pos < seq_len; ++pos) { - const float pos_float = static_cast(pos); - for (size_t i = 0; i < half_dim; ++i) { - const float freq = 1.0f / powf(theta, (2.0f * i) / head_dim); - const float angle = pos_float * freq; - - const size_t idx = pos * half_dim + i; - rope_cache_f16.cos_table[idx] = static_cast<__fp16>(cosf(angle)); - rope_cache_f16.sin_table[idx] = static_cast<__fp16>(sinf(angle)); - } - } - - rope_cache_f16.max_seq_len = seq_len; - rope_cache_f16.head_dim = head_dim; - rope_cache_f16.theta = theta; - rope_cache_f16.initialized = true; -} - -} - -void cactus_rope_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t num_heads, - size_t head_dim, - size_t start_pos, - float theta -) { - const size_t half_dim = head_dim / 2; - - CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, head_dim, theta); - - const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_dim; - const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_dim; - - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t batch_idx = idx / seq_len; - const size_t seq_idx = idx % seq_len; - - for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { - const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; - const __fp16* input_ptr = input + offset; - __fp16* output_ptr = output + offset; - - const __fp16* cos_ptr = cos_cache + seq_idx * half_dim; - const __fp16* sin_ptr = sin_cache + seq_idx * half_dim; - - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_half_dim = (half_dim / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vectorized_half_dim; i += SIMD_WIDTH) { - float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); - float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); - - float16x8_t x_first_half = vld1q_f16(&input_ptr[i]); - float16x8_t x_second_half = vld1q_f16(&input_ptr[i + half_dim]); - - float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); - float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); - - vst1q_f16(&output_ptr[i], first_result); - vst1q_f16(&output_ptr[i + half_dim], second_result); - } - - for (size_t i = vectorized_half_dim; i < half_dim; ++i) { - const __fp16 cos_val = cos_ptr[i]; - const __fp16 sin_val = sin_ptr[i]; - - const __fp16 x_first_half = input_ptr[i]; - const __fp16 x_second_half = input_ptr[i + half_dim]; - - output_ptr[i] = x_first_half * cos_val - x_second_half * sin_val; - - output_ptr[i + half_dim] = x_second_half * cos_val + x_first_half * sin_val; - } - } - } - }); -} - -void cactus_gpt_j_rope_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t num_heads, - size_t head_dim, - size_t rot_dim, - size_t start_pos, - float theta -) { - const size_t half_rot_dim = rot_dim / 2; - - CactusRoPEF16::precompute_rope_tables_f16(seq_len + start_pos, rot_dim, theta); - - const __fp16* cos_cache = CactusRoPEF16::rope_cache_f16.cos_table.data() + start_pos * half_rot_dim; - const __fp16* sin_cache = CactusRoPEF16::rope_cache_f16.sin_table.data() + start_pos * half_rot_dim; - - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t batch_idx = idx / seq_len; - const size_t seq_idx = idx % seq_len; - - for (size_t head_idx = 0; head_idx < num_heads; ++head_idx) { - const size_t offset = ((batch_idx * seq_len + seq_idx) * num_heads + head_idx) * head_dim; - const __fp16* input_ptr = input + offset; - __fp16* output_ptr = output + offset; - - const __fp16* cos_ptr = cos_cache + seq_idx * half_rot_dim; - const __fp16* sin_ptr = sin_cache + seq_idx * half_rot_dim; - - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_half_rot_dim = (half_rot_dim / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vectorized_half_rot_dim; i += SIMD_WIDTH) { - float16x8_t cos_vec = vld1q_f16(&cos_ptr[i]); - float16x8_t sin_vec = vld1q_f16(&sin_ptr[i]); - - float16x8x2_t x_vec = vld2q_f16(&input_ptr[2*i]); - float16x8_t x_first_half = x_vec.val[0]; - float16x8_t x_second_half = x_vec.val[1]; - - float16x8_t first_result = vfmsq_f16(vmulq_f16(x_first_half, cos_vec), x_second_half, sin_vec); - float16x8_t second_result = vfmaq_f16(vmulq_f16(x_second_half, cos_vec), x_first_half, sin_vec); - - float16x8x2_t t; - t.val[0] = first_result; - t.val[1] = second_result; - vst2q_f16(&output_ptr[2*i], t); - } - - for (size_t i = vectorized_half_rot_dim; i < half_rot_dim; ++i) { - const __fp16 cos_val = cos_ptr[i]; - const __fp16 sin_val = sin_ptr[i]; - - const __fp16 x_first_half = input_ptr[2*i]; - const __fp16 x_second_half = input_ptr[2*i + 1]; - - output_ptr[2*i] = x_first_half * cos_val - x_second_half * sin_val; - - output_ptr[2*i + 1] = x_second_half * cos_val + x_first_half * sin_val; - } - - constexpr size_t TAIL_SIMD_WIDTH = 8; - size_t copy_idx = rot_dim; - const size_t copy_end_vec = (head_dim / TAIL_SIMD_WIDTH) * TAIL_SIMD_WIDTH; - - for (; copy_idx + TAIL_SIMD_WIDTH <= copy_end_vec; copy_idx += TAIL_SIMD_WIDTH) { - float16x8_t v = vld1q_f16(&input_ptr[copy_idx]); - vst1q_f16(&output_ptr[copy_idx], v); - } - for (; copy_idx < head_dim; ++copy_idx) { - output_ptr[copy_idx] = input_ptr[copy_idx]; - } - } - } - }); -} diff --git a/cactus/kernel/kernel_bilstm.cpp b/cactus/kernel/kernel_bilstm.cpp deleted file mode 100644 index 2b4ea2321..000000000 --- a/cactus/kernel/kernel_bilstm.cpp +++ /dev/null @@ -1,370 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -static void apply_lstm_gates_f32( - const float* __restrict gates, - float* __restrict c, - float* __restrict h, - size_t hidden_size -) { - const float32x4_t one = vdupq_n_f32(1.0f); - size_t i = 0; - - for (; i + 4 <= hidden_size; i += 4) { - float32x4_t i_raw = vld1q_f32(gates + i); - float32x4_t f_raw = vld1q_f32(gates + hidden_size + i); - float32x4_t g_raw = vld1q_f32(gates + 2 * hidden_size + i); - float32x4_t o_raw = vld1q_f32(gates + 3 * hidden_size + i); - - float32x4_t i_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_raw)))); - float32x4_t f_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_raw)))); - float32x4_t g_act = fast_tanh_f32x4(g_raw); - float32x4_t o_act = vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_raw)))); - - float32x4_t c_prev = vld1q_f32(c + i); - float32x4_t c_new = vfmaq_f32(vmulq_f32(f_act, c_prev), i_act, g_act); - vst1q_f32(c + i, c_new); - - vst1q_f32(h + i, vmulq_f32(o_act, fast_tanh_f32x4(c_new))); - } - - for (; i < hidden_size; ++i) { - float ig = 1.0f / (1.0f + expf(-gates[i])); - float fg = 1.0f / (1.0f + expf(-gates[hidden_size + i])); - float gg = tanhf(gates[2 * hidden_size + i]); - float og = 1.0f / (1.0f + expf(-gates[3 * hidden_size + i])); - float cv = fg * c[i] + ig * gg; - c[i] = cv; - h[i] = og * tanhf(cv); - } -} - -#ifndef __APPLE__ -static void apply_lstm_gates_f16( - const __fp16* __restrict gates, - __fp16* __restrict c, - __fp16* __restrict h, - size_t hidden_size -) { - constexpr size_t W = 8; - const size_t simd_end = (hidden_size / W) * W; - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t i = 0; i < simd_end; i += W) { - float16x8_t i_raw = vld1q_f16(gates + i); - float16x8_t f_raw = vld1q_f16(gates + hidden_size + i); - float16x8_t g_raw = vld1q_f16(gates + 2 * hidden_size + i); - float16x8_t o_raw = vld1q_f16(gates + 3 * hidden_size + i); - - float32x4_t il = vcvt_f32_f16(vget_low_f16(i_raw)); - float32x4_t ih = vcvt_f32_f16(vget_high_f16(i_raw)); - float16x8_t i_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(il))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ih)))))); - - float32x4_t fl = vcvt_f32_f16(vget_low_f16(f_raw)); - float32x4_t fh = vcvt_f32_f16(vget_high_f16(f_raw)); - float16x8_t f_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fl))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(fh)))))); - - float32x4_t gl = vcvt_f32_f16(vget_low_f16(g_raw)); - float32x4_t gh = vcvt_f32_f16(vget_high_f16(g_raw)); - float16x8_t g_act = vcombine_f16( - vcvt_f16_f32(fast_tanh_f32x4(gl)), - vcvt_f16_f32(fast_tanh_f32x4(gh))); - - float32x4_t ol = vcvt_f32_f16(vget_low_f16(o_raw)); - float32x4_t oh = vcvt_f32_f16(vget_high_f16(o_raw)); - float16x8_t o_act = vcombine_f16( - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(ol))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(oh)))))); - - float16x8_t c_prev = vld1q_f16(c + i); - float16x8_t c_new = vfmaq_f16(vmulq_f16(f_act, c_prev), i_act, g_act); - vst1q_f16(c + i, c_new); - - float32x4_t cl = vcvt_f32_f16(vget_low_f16(c_new)); - float32x4_t ch = vcvt_f32_f16(vget_high_f16(c_new)); - float16x8_t c_tanh = vcombine_f16( - vcvt_f16_f32(fast_tanh_f32x4(cl)), - vcvt_f16_f32(fast_tanh_f32x4(ch))); - vst1q_f16(h + i, vmulq_f16(o_act, c_tanh)); - } - - for (size_t i = simd_end; i < hidden_size; ++i) { - float ig = 1.0f / (1.0f + expf(-static_cast(gates[i]))); - float fg = 1.0f / (1.0f + expf(-static_cast(gates[hidden_size + i]))); - float gg = tanhf(static_cast(gates[2 * hidden_size + i])); - float og = 1.0f / (1.0f + expf(-static_cast(gates[3 * hidden_size + i]))); - float cv = fg * static_cast(c[i]) + ig * gg; - c[i] = static_cast<__fp16>(cv); - h[i] = static_cast<__fp16>(og * tanhf(cv)); - } -} - -static inline float hsum_f16x8_f32(float16x8_t v) { - float16x4_t lo = vget_low_f16(v); - float16x4_t hi = vget_high_f16(v); - float16x4_t s4 = vadd_f16(lo, hi); - float16x4_t s2 = vadd_f16(s4, vext_f16(s4, s4, 2)); - float16x4_t s1 = vadd_f16(s2, vext_f16(s2, s2, 1)); - return static_cast(vget_lane_f16(s1, 0)); -} - -static void lstm_gemv_f16_neon( - const __fp16* __restrict x, - const __fp16* __restrict W, - const __fp16* __restrict bias, - __fp16* __restrict out, - size_t K, size_t N -) { - const size_t K16 = (K / 16) * 16; - const size_t K8 = (K / 8) * 8; - size_t n = 0; - for (; n + 4 <= N; n += 4) { - const __fp16* w0 = W + n * K; - const __fp16* w1 = W + (n + 1) * K; - const __fp16* w2 = W + (n + 2) * K; - const __fp16* w3 = W + (n + 3) * K; - float16x8_t a0 = vdupq_n_f16(0); - float16x8_t a1 = vdupq_n_f16(0); - float16x8_t a2 = vdupq_n_f16(0); - float16x8_t a3 = vdupq_n_f16(0); - for (size_t k = 0; k < K16; k += 16) { - float16x8_t xlo = vld1q_f16(x + k); - float16x8_t xhi = vld1q_f16(x + k + 8); - a0 = vfmaq_f16(a0, xlo, vld1q_f16(w0 + k)); - a0 = vfmaq_f16(a0, xhi, vld1q_f16(w0 + k + 8)); - a1 = vfmaq_f16(a1, xlo, vld1q_f16(w1 + k)); - a1 = vfmaq_f16(a1, xhi, vld1q_f16(w1 + k + 8)); - a2 = vfmaq_f16(a2, xlo, vld1q_f16(w2 + k)); - a2 = vfmaq_f16(a2, xhi, vld1q_f16(w2 + k + 8)); - a3 = vfmaq_f16(a3, xlo, vld1q_f16(w3 + k)); - a3 = vfmaq_f16(a3, xhi, vld1q_f16(w3 + k + 8)); - } - for (size_t k = K16; k < K8; k += 8) { - float16x8_t xv = vld1q_f16(x + k); - a0 = vfmaq_f16(a0, xv, vld1q_f16(w0 + k)); - a1 = vfmaq_f16(a1, xv, vld1q_f16(w1 + k)); - a2 = vfmaq_f16(a2, xv, vld1q_f16(w2 + k)); - a3 = vfmaq_f16(a3, xv, vld1q_f16(w3 + k)); - } - float s0 = hsum_f16x8_f32(a0); - float s1 = hsum_f16x8_f32(a1); - float s2 = hsum_f16x8_f32(a2); - float s3 = hsum_f16x8_f32(a3); - for (size_t k = K8; k < K; ++k) { - float xv = static_cast(x[k]); - s0 += xv * static_cast(w0[k]); - s1 += xv * static_cast(w1[k]); - s2 += xv * static_cast(w2[k]); - s3 += xv * static_cast(w3[k]); - } - out[n] = static_cast<__fp16>(s0 + static_cast(bias[n])); - out[n + 1] = static_cast<__fp16>(s1 + static_cast(bias[n + 1])); - out[n + 2] = static_cast<__fp16>(s2 + static_cast(bias[n + 2])); - out[n + 3] = static_cast<__fp16>(s3 + static_cast(bias[n + 3])); - } - for (; n < N; ++n) { - const __fp16* w = W + n * K; - float16x8_t acc = vdupq_n_f16(0); - for (size_t k = 0; k < K8; k += 8) - acc = vfmaq_f16(acc, vld1q_f16(x + k), vld1q_f16(w + k)); - float s = hsum_f16x8_f32(acc); - for (size_t k = K8; k < K; ++k) - s += static_cast(x[k]) * static_cast(w[k]); - out[n] = static_cast<__fp16>(s + static_cast(bias[n])); - } -} -#endif - -static inline void fp16_to_fp32_neon(const __fp16* src, float* dst, size_t n) { - size_t i = 0; - for (; i + 8 <= n; i += 8) { - float16x8_t v = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; i < n; ++i) - dst[i] = static_cast(src[i]); -} - -static inline void fp32_to_fp16_neon(const float* src, __fp16* dst, size_t n) { - size_t i = 0; - for (; i + 8 <= n; i += 8) { - float32x4_t lo = vld1q_f32(src + i); - float32x4_t hi = vld1q_f32(src + i + 4); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi))); - } - for (; i < n; ++i) - dst[i] = static_cast<__fp16>(src[i]); -} - -void cactus_bilstm_sequence_f16( - const __fp16* input, - const __fp16* weight_ih_fwd, - const __fp16* weight_hh_fwd, - const __fp16* bias_ih_fwd, - const __fp16* bias_hh_fwd, - const __fp16* weight_ih_bwd, - const __fp16* weight_hh_bwd, - const __fp16* bias_ih_bwd, - const __fp16* bias_hh_bwd, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t input_size, - size_t hidden_size -) { - const size_t gate_size = 4 * hidden_size; - const size_t combined_K = input_size + hidden_size; - const size_t output_size = 2 * hidden_size; - - std::vector bias_fwd_f32(gate_size); - std::vector bias_bwd_f32(gate_size); - for (size_t g = 0; g < gate_size; ++g) { - bias_fwd_f32[g] = static_cast(bias_ih_fwd[g]) + static_cast(bias_hh_fwd[g]); - bias_bwd_f32[g] = static_cast(bias_ih_bwd[g]) + static_cast(bias_hh_bwd[g]); - } - -#ifdef __APPLE__ - std::vector W_fwd_f32(gate_size * combined_K); - std::vector W_bwd_f32(gate_size * combined_K); - for (size_t g = 0; g < gate_size; ++g) { - fp16_to_fp32_neon(weight_ih_fwd + g * input_size, - W_fwd_f32.data() + g * combined_K, input_size); - fp16_to_fp32_neon(weight_hh_fwd + g * hidden_size, - W_fwd_f32.data() + g * combined_K + input_size, hidden_size); - fp16_to_fp32_neon(weight_ih_bwd + g * input_size, - W_bwd_f32.data() + g * combined_K, input_size); - fp16_to_fp32_neon(weight_hh_bwd + g * hidden_size, - W_bwd_f32.data() + g * combined_K + input_size, hidden_size); - } - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* batch_in = input + b * seq_len * input_size; - __fp16* batch_out = output + b * seq_len * output_size; - - auto& pool = CactusThreading::get_thread_pool(); - auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { - std::vector xh(combined_K); - std::vector gates(gate_size); - std::vector h(hidden_size, 0.0f); - std::vector c(hidden_size, 0.0f); - - for (size_t idx = 0; idx < seq_len; ++idx) { - const size_t t = seq_len - 1 - idx; - fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); - - memcpy(gates.data(), bias_bwd_f32.data(), gate_size * sizeof(float)); - cblas_sgemv(CblasRowMajor, CblasNoTrans, - static_cast(gate_size), static_cast(combined_K), - 1.0f, W_bwd_f32.data(), static_cast(combined_K), - xh.data(), 1, 1.0f, gates.data(), 1); - - apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); - - fp32_to_fp16_neon(h.data(), batch_out + t * output_size + hidden_size, hidden_size); - } - }); - - { - std::vector xh(combined_K); - std::vector gates(gate_size); - std::vector h(hidden_size, 0.0f); - std::vector c(hidden_size, 0.0f); - - for (size_t t = 0; t < seq_len; ++t) { - fp16_to_fp32_neon(batch_in + t * input_size, xh.data(), input_size); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(float)); - - memcpy(gates.data(), bias_fwd_f32.data(), gate_size * sizeof(float)); - cblas_sgemv(CblasRowMajor, CblasNoTrans, - static_cast(gate_size), static_cast(combined_K), - 1.0f, W_fwd_f32.data(), static_cast(combined_K), - xh.data(), 1, 1.0f, gates.data(), 1); - - apply_lstm_gates_f32(gates.data(), c.data(), h.data(), hidden_size); - - fp32_to_fp16_neon(h.data(), batch_out + t * output_size, hidden_size); - } - } - - bwd_future.get(); - } - -#else - std::vector<__fp16> W_fwd_f16(gate_size * combined_K); - std::vector<__fp16> W_bwd_f16(gate_size * combined_K); - for (size_t g = 0; g < gate_size; ++g) { - memcpy(W_fwd_f16.data() + g * combined_K, - weight_ih_fwd + g * input_size, input_size * sizeof(__fp16)); - memcpy(W_fwd_f16.data() + g * combined_K + input_size, - weight_hh_fwd + g * hidden_size, hidden_size * sizeof(__fp16)); - memcpy(W_bwd_f16.data() + g * combined_K, - weight_ih_bwd + g * input_size, input_size * sizeof(__fp16)); - memcpy(W_bwd_f16.data() + g * combined_K + input_size, - weight_hh_bwd + g * hidden_size, hidden_size * sizeof(__fp16)); - } - - std::vector<__fp16> bias_fwd_f16(gate_size); - std::vector<__fp16> bias_bwd_f16(gate_size); - for (size_t g = 0; g < gate_size; ++g) { - bias_fwd_f16[g] = static_cast<__fp16>(bias_fwd_f32[g]); - bias_bwd_f16[g] = static_cast<__fp16>(bias_bwd_f32[g]); - } - - for (size_t b = 0; b < batch_size; ++b) { - const __fp16* batch_in = input + b * seq_len * input_size; - __fp16* batch_out = output + b * seq_len * output_size; - - auto& pool = CactusThreading::get_thread_pool(); - auto bwd_future = pool.enqueue([&, batch_in, batch_out]() { - std::vector<__fp16> xh(combined_K); - std::vector<__fp16> gates(gate_size); - std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); - std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); - - for (size_t idx = 0; idx < seq_len; ++idx) { - const size_t t = seq_len - 1 - idx; - memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); - lstm_gemv_f16_neon(xh.data(), W_bwd_f16.data(), bias_bwd_f16.data(), - gates.data(), combined_K, gate_size); - apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); - memcpy(batch_out + t * output_size + hidden_size, - h.data(), hidden_size * sizeof(__fp16)); - } - }); - - { - std::vector<__fp16> xh(combined_K); - std::vector<__fp16> gates(gate_size); - std::vector<__fp16> h(hidden_size, static_cast<__fp16>(0)); - std::vector<__fp16> c(hidden_size, static_cast<__fp16>(0)); - - for (size_t t = 0; t < seq_len; ++t) { - memcpy(xh.data(), batch_in + t * input_size, input_size * sizeof(__fp16)); - memcpy(xh.data() + input_size, h.data(), hidden_size * sizeof(__fp16)); - lstm_gemv_f16_neon(xh.data(), W_fwd_f16.data(), bias_fwd_f16.data(), - gates.data(), combined_K, gate_size); - apply_lstm_gates_f16(gates.data(), c.data(), h.data(), hidden_size); - memcpy(batch_out + t * output_size, - h.data(), hidden_size * sizeof(__fp16)); - } - } - - bwd_future.get(); - } -#endif -} diff --git a/cactus/kernel/kernel_blas.cpp b/cactus/kernel/kernel_blas.cpp deleted file mode 100644 index 2d6f9ba72..000000000 --- a/cactus/kernel/kernel_blas.cpp +++ /dev/null @@ -1,591 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include - -static inline size_t compute_linear_index(const size_t* coords, const size_t* strides, size_t ndim) { - size_t index = 0; - for (size_t i = 0; i < ndim; ++i) { - index += coords[i] * strides[i]; - } - return index; -} - -static inline void increment_coords(size_t* coords, const size_t* shape, size_t ndim) { - for (int i = ndim - 1; i >= 0; --i) { - coords[i]++; - if (coords[i] < shape[i]) { - break; - } - coords[i] = 0; - } -} - -enum class BroadcastOp { ADD, SUB, MUL, DIV }; - -template -static inline float16x8_t broadcast_op_vec(float16x8_t a, float16x8_t b) { - if constexpr (Op == BroadcastOp::ADD) return vaddq_f16(a, b); - else if constexpr (Op == BroadcastOp::SUB) return vsubq_f16(a, b); - else if constexpr (Op == BroadcastOp::MUL) return vmulq_f16(a, b); - else return vdivq_f16(a, b); -} - -template -static inline __fp16 broadcast_op_scalar(__fp16 a, __fp16 b) { - if constexpr (Op == BroadcastOp::ADD) return a + b; - else if constexpr (Op == BroadcastOp::SUB) return a - b; - else if constexpr (Op == BroadcastOp::MUL) return a * b; - else return a / b; -} - -template -static void broadcast_op_optimized(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - size_t total_elements = 1; - for (size_t i = 0; i < ndim; ++i) { - total_elements *= output_shape[i]; - } - - if (total_elements == 0) return; - - size_t inner_size = output_shape[ndim - 1]; - bool a_inner_contiguous = (a_strides[ndim - 1] == 1) || (a_strides[ndim - 1] == 0); - bool b_inner_contiguous = (b_strides[ndim - 1] == 1) || (b_strides[ndim - 1] == 0); - bool a_inner_broadcast = (a_strides[ndim - 1] == 0); - bool b_inner_broadcast = (b_strides[ndim - 1] == 0); - - size_t outer_size = total_elements / inner_size; - - if (a_inner_contiguous && b_inner_contiguous && inner_size >= 8) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_outer, size_t end_outer) { - std::vector coords(ndim, 0); - - size_t tmp = start_outer; - for (int i = ndim - 2; i >= 0; --i) { - coords[i] = tmp % output_shape[i]; - tmp /= output_shape[i]; - } - - for (size_t outer_idx = start_outer; outer_idx < end_outer; ++outer_idx) { - size_t a_base = 0, b_base = 0; - for (size_t i = 0; i < ndim - 1; ++i) { - a_base += coords[i] * a_strides[i]; - b_base += coords[i] * b_strides[i]; - } - - __fp16* out_ptr = output + outer_idx * inner_size; - const size_t vec_end = (inner_size / 8) * 8; - - const bool use_stream = total_elements >= STREAMING_STORE_THRESHOLD; - - if (a_inner_broadcast && b_inner_broadcast) { - __fp16 result = broadcast_op_scalar(a[a_base], b[b_base]); - float16x8_t result_vec = vdupq_n_f16(result); - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - stream_store_f16x8(out_ptr + i, result_vec); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - vst1q_f16(out_ptr + i, result_vec); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = result; - } - } else if (a_inner_broadcast) { - float16x8_t a_vec = vdupq_n_f16(a[a_base]); - const __fp16* b_ptr = b + b_base; - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t b_vec = vld1q_f16(b_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t b_vec = vld1q_f16(b_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a[a_base], b_ptr[i]); - } - } else if (b_inner_broadcast) { - const __fp16* a_ptr = a + a_base; - float16x8_t b_vec = vdupq_n_f16(b[b_base]); - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a_ptr[i], b[b_base]); - } - } else { - const __fp16* a_ptr = a + a_base; - const __fp16* b_ptr = b + b_base; - if (use_stream) { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - float16x8_t b_vec = vld1q_f16(b_ptr + i); - stream_store_f16x8(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } else { - for (size_t i = 0; i < vec_end; i += 8) { - float16x8_t a_vec = vld1q_f16(a_ptr + i); - float16x8_t b_vec = vld1q_f16(b_ptr + i); - vst1q_f16(out_ptr + i, broadcast_op_vec(a_vec, b_vec)); - } - } - for (size_t i = vec_end; i < inner_size; ++i) { - out_ptr[i] = broadcast_op_scalar(a_ptr[i], b_ptr[i]); - } - } - - for (int i = ndim - 2; i >= 0; --i) { - coords[i]++; - if (coords[i] < output_shape[i]) break; - coords[i] = 0; - } - } - }); - } else { - CactusThreading::parallel_for(total_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - std::vector coords(ndim); - - size_t tmp = start_idx; - for (int i = ndim - 1; i >= 0; --i) { - coords[i] = tmp % output_shape[i]; - tmp /= output_shape[i]; - } - - for (size_t linear_idx = start_idx; linear_idx < end_idx; ++linear_idx) { - size_t a_idx = compute_linear_index(coords.data(), a_strides, ndim); - size_t b_idx = compute_linear_index(coords.data(), b_strides, ndim); - - output[linear_idx] = broadcast_op_scalar(a[a_idx], b[b_idx]); - - increment_coords(coords.data(), output_shape, ndim); - } - }); - } -} - -void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL = 4; - const size_t unrolled_end = start_idx + ((end_idx - start_idx) / (SIMD_WIDTH * UNROLL)) * (SIMD_WIDTH * UNROLL); - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < unrolled_end; i += SIMD_WIDTH * UNROLL) { - __builtin_prefetch(&a[i + 256], 0, 0); - __builtin_prefetch(&b[i + 256], 0, 0); - float16x8_t v0 = vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i])); - float16x8_t v1 = vaddq_f16(vld1q_f16(&a[i + 8]), vld1q_f16(&b[i + 8])); - float16x8_t v2 = vaddq_f16(vld1q_f16(&a[i + 16]), vld1q_f16(&b[i + 16])); - float16x8_t v3 = vaddq_f16(vld1q_f16(&a[i + 24]), vld1q_f16(&b[i + 24])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - stream_store_f16x8(&output[i + 16], v2); - stream_store_f16x8(&output[i + 24], v3); - } - for (size_t i = unrolled_end; i < vectorized_end; i += SIMD_WIDTH) { - stream_store_f16x8(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - vst1q_f16(&output[i], vaddq_f16(vld1q_f16(&a[i]), vld1q_f16(&b[i]))); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] + b[i]; - } - }); -} - -void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - constexpr float FP16_MAX = 65500.0f; - const float32x4_t max_val = vdupq_n_f32(FP16_MAX); - const float32x4_t min_val = vdupq_n_f32(-FP16_MAX); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - - float32x4_t a_low = vcvt_f32_f16(vget_low_f16(a_vec)); - float32x4_t a_high = vcvt_f32_f16(vget_high_f16(a_vec)); - float32x4_t b_low = vcvt_f32_f16(vget_low_f16(b_vec)); - float32x4_t b_high = vcvt_f32_f16(vget_high_f16(b_vec)); - - float32x4_t result_low = vaddq_f32(a_low, b_low); - float32x4_t result_high = vaddq_f32(a_high, b_high); - - result_low = vminq_f32(vmaxq_f32(result_low, min_val), max_val); - result_high = vminq_f32(vmaxq_f32(result_high, min_val), max_val); - - float16x4_t result_low_f16 = vcvt_f16_f32(result_low); - float16x4_t result_high_f16 = vcvt_f16_f32(result_high); - float16x8_t result_vec = vcombine_f16(result_low_f16, result_high_f16); - - vst1q_f16(&output[i], result_vec); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float result = static_cast(a[i]) + static_cast(b[i]); - result = std::fmin(std::fmax(result, -FP16_MAX), FP16_MAX); - output[i] = static_cast<__fp16>(result); - } - }); -} - -void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vsubq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vsubq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] - b[i]; - } - }); -} - -void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vmulq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vmulq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] * b[i]; - } - }); -} - -void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t num_elements, float scale) { - constexpr size_t SIMD_WIDTH = 8; - const float32x4_t vscale = vdupq_n_f32(scale); - const size_t vec_end = (num_elements / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = 0; i < vec_end; i += SIMD_WIDTH) { - float16x8_t base_vec = vld1q_f16(base + i); - float16x8_t src_vec = vld1q_f16(src + i); - - float32x4_t base_lo = vcvt_f32_f16(vget_low_f16(base_vec)); - float32x4_t base_hi = vcvt_f32_f16(vget_high_f16(base_vec)); - float32x4_t src_lo = vcvt_f32_f16(vget_low_f16(src_vec)); - float32x4_t src_hi = vcvt_f32_f16(vget_high_f16(src_vec)); - - float32x4_t result_lo = vfmaq_f32(base_lo, src_lo, vscale); - float32x4_t result_hi = vfmaq_f32(base_hi, src_hi, vscale); - - vst1q_f16(output + i, vcombine_f16(vcvt_f16_f32(result_lo), vcvt_f16_f32(result_hi))); - } - for (size_t i = vec_end; i < num_elements; ++i) { - output[i] = static_cast<__fp16>(static_cast(base[i]) - + static_cast(src[i]) * scale); - } -} - -void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t num_elements) { - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - if (use_streaming) { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - stream_store_f16x8(&output[i], vdivq_f16(a_vec, b_vec)); - } - } else { - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t a_vec = vld1q_f16(&a[i]); - float16x8_t b_vec = vld1q_f16(&b[i]); - vst1q_f16(&output[i], vdivq_f16(a_vec, b_vec)); - } - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - output[i] = a[i] / b[i]; - } - }); -} - -void cactus_add_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_subtract_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_multiply_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_divide_broadcast_f16(const __fp16* a, const __fp16* b, __fp16* output, - const size_t* a_strides, const size_t* b_strides, - const size_t* output_shape, size_t ndim) { - broadcast_op_optimized(a, b, output, a_strides, b_strides, output_shape, ndim); -} - -void cactus_concat_f16(const __fp16* input1, const __fp16* input2, __fp16* output, - const size_t* shape1, const size_t* shape2, const size_t* output_shape, - size_t ndims, int axis) { - if (axis < 0) axis += ndims; - - size_t outer_size = 1; - for (size_t i = 0; i < static_cast(axis); ++i) { - outer_size *= output_shape[i]; - } - - size_t inner_size = 1; - for (size_t i = axis + 1; i < ndims; ++i) { - inner_size *= output_shape[i]; - } - - size_t axis_size1 = shape1[axis]; - size_t axis_size2 = shape2[axis]; - - size_t input1_stride = axis_size1 * inner_size; - size_t input2_stride = axis_size2 * inner_size; - size_t output_stride = (axis_size1 + axis_size2) * inner_size; - - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t outer = start; outer < end; ++outer) { - const __fp16* in1_ptr = input1 + outer * input1_stride; - const __fp16* in2_ptr = input2 + outer * input2_stride; - __fp16* out_ptr = output + outer * output_stride; - - size_t copy_size1 = axis_size1 * inner_size; - std::memcpy(out_ptr, in1_ptr, copy_size1 * sizeof(__fp16)); - - size_t copy_size2 = axis_size2 * inner_size; - std::memcpy(out_ptr + copy_size1, in2_ptr, copy_size2 * sizeof(__fp16)); - } - }); -} - -void cactus_cat_f16(const __fp16** inputs, __fp16* output, const size_t** input_shapes, - const size_t* output_shape, size_t num_inputs, size_t rank, int axis) { - if (axis < 0) axis += rank; - - size_t outer_size = 1; - for (size_t i = 0; i < static_cast(axis); ++i) { - outer_size *= output_shape[i]; - } - - size_t inner_size = 1; - for (size_t i = axis + 1; i < rank; ++i) { - inner_size *= output_shape[i]; - } - - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t outer = start; outer < end; ++outer) { - __fp16* out_ptr = output + outer * inner_size * output_shape[axis]; - size_t offset = 0; - - for (size_t input_idx = 0; input_idx < num_inputs; ++input_idx) { - const __fp16* in_ptr = inputs[input_idx] + outer * inner_size * input_shapes[input_idx][axis]; - size_t copy_size = input_shapes[input_idx][axis] * inner_size; - std::memcpy(out_ptr + offset, in_ptr, copy_size * sizeof(__fp16)); - offset += copy_size; - } - } - }); -} - -void cactus_transpose_2d_f16(const __fp16* source, __fp16* destination, size_t num_rows, size_t num_cols, size_t start_row, size_t end_row) { - constexpr size_t TILE_SIZE = 32; - constexpr size_t VECTOR_WIDTH = 8; - - for (size_t row_tile_start = start_row; row_tile_start < end_row; row_tile_start += TILE_SIZE) { - const size_t row_tile_end = std::min(row_tile_start + TILE_SIZE, end_row); - - for (size_t col_tile_start = 0; col_tile_start < num_cols; col_tile_start += TILE_SIZE) { - const size_t col_tile_end = std::min(col_tile_start + TILE_SIZE, num_cols); - - for (size_t row_block = row_tile_start; row_block < row_tile_end; row_block += VECTOR_WIDTH) { - const size_t row_block_end = std::min(row_block + VECTOR_WIDTH, row_tile_end); - - for (size_t col_block = col_tile_start; col_block < col_tile_end; col_block += VECTOR_WIDTH) { - const size_t col_block_end = std::min(col_block + VECTOR_WIDTH, col_tile_end); - - if (row_block_end - row_block >= 8 && col_block_end - col_block >= 8) { - float16x8_t rows[8]; - for (int i = 0; i < 8; i++) { - if (row_block + i < row_block_end) { - rows[i] = vld1q_f16(&source[(row_block + i) * num_cols + col_block]); - } else { - rows[i] = vdupq_n_f16(0.0f); - } - } - - float16x8x2_t r01 = vtrnq_f16(rows[0], rows[1]); - float16x8x2_t r23 = vtrnq_f16(rows[2], rows[3]); - float16x8x2_t r45 = vtrnq_f16(rows[4], rows[5]); - float16x8x2_t r67 = vtrnq_f16(rows[6], rows[7]); - - float32x4x2_t r0123_lo = vtrnq_f32( - vreinterpretq_f32_f16(r01.val[0]), - vreinterpretq_f32_f16(r23.val[0])); - float32x4x2_t r0123_hi = vtrnq_f32( - vreinterpretq_f32_f16(r01.val[1]), - vreinterpretq_f32_f16(r23.val[1])); - float32x4x2_t r4567_lo = vtrnq_f32( - vreinterpretq_f32_f16(r45.val[0]), - vreinterpretq_f32_f16(r67.val[0])); - float32x4x2_t r4567_hi = vtrnq_f32( - vreinterpretq_f32_f16(r45.val[1]), - vreinterpretq_f32_f16(r67.val[1])); - - float16x8_t col0 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_lo.val[0]), vget_low_f32(r4567_lo.val[0]))); - float16x8_t col1 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_hi.val[0]), vget_low_f32(r4567_hi.val[0]))); - float16x8_t col2 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_lo.val[1]), vget_low_f32(r4567_lo.val[1]))); - float16x8_t col3 = vreinterpretq_f16_f32(vcombine_f32( - vget_low_f32(r0123_hi.val[1]), vget_low_f32(r4567_hi.val[1]))); - float16x8_t col4 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_lo.val[0]), vget_high_f32(r4567_lo.val[0]))); - float16x8_t col5 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_hi.val[0]), vget_high_f32(r4567_hi.val[0]))); - float16x8_t col6 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_lo.val[1]), vget_high_f32(r4567_lo.val[1]))); - float16x8_t col7 = vreinterpretq_f16_f32(vcombine_f32( - vget_high_f32(r0123_hi.val[1]), vget_high_f32(r4567_hi.val[1]))); - - float16x8_t cols[8] = {col0, col1, col2, col3, col4, col5, col6, col7}; - for (int c = 0; c < 8 && col_block + c < col_block_end; c++) { - if (col_block + c < num_cols) { - if (row_block_end - row_block >= 8) { - vst1q_f16(&destination[(col_block + c) * num_rows + row_block], cols[c]); - } else { - __fp16 temp[8]; - vst1q_f16(temp, cols[c]); - for (size_t i = 0; i < row_block_end - row_block; ++i) { - destination[(col_block + c) * num_rows + row_block + i] = temp[i]; - } - } - } - } - } else { - for (size_t row = row_block; row < row_block_end; row++) { - for (size_t col = col_block; col < col_block_end; col++) { - destination[col * num_rows + row] = source[row * num_cols + col]; - } - } - } - } - } - } - } -} - -void cactus_transpose_f16(const __fp16* source, __fp16* destination, const size_t* shape, const size_t* permutation, size_t ndim, size_t start_idx, size_t end_idx) { - if (ndim == 2 && permutation[0] == 1 && permutation[1] == 0) { - size_t num_rows = shape[0]; - size_t num_cols = shape[1]; - - constexpr size_t THRESHOLD = 8192; - constexpr size_t TILE_ROWS = 32; - if (num_rows * num_cols >= THRESHOLD) { - const size_t num_row_blocks = (num_rows + TILE_ROWS - 1) / TILE_ROWS; - - CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start_block, size_t end_block) { - for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { - size_t start_row = block_idx * TILE_ROWS; - size_t end_row = std::min(start_row + TILE_ROWS, num_rows); - - cactus_transpose_2d_f16(source, destination, num_rows, num_cols, start_row, end_row); - } - }); - } else { - cactus_transpose_2d_f16(source, destination, num_rows, num_cols, 0, num_rows); - } - } else { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - size_t src_idx = 0; - size_t tmp_idx = idx; - - for (size_t i = 0; i < ndim; ++i) { - size_t coord = tmp_idx % shape[permutation[ndim - 1 - i]]; - tmp_idx /= shape[permutation[ndim - 1 - i]]; - - size_t stride = 1; - for (size_t j = permutation[ndim - 1 - i] + 1; j < ndim; ++j) { - stride *= shape[j]; - } - src_idx += coord * stride; - } - - destination[idx] = source[src_idx]; - } - } -} diff --git a/cactus/kernel/kernel_conv.cpp b/cactus/kernel/kernel_conv.cpp deleted file mode 100644 index 7e7dbe5f1..000000000 --- a/cactus/kernel/kernel_conv.cpp +++ /dev/null @@ -1,1581 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#endif - -constexpr size_t T_TILE_F16 = 2; -constexpr size_t ACCELERATE_K_THRESHOLD = 32; -constexpr size_t ACCELERATE_L_THRESHOLD = 128; - -#ifdef __APPLE__ -static void conv1d_causal_depthwise_f16_accelerate( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, size_t C, size_t K, size_t dilation) -{ - const size_t in_bs = L * C; - const size_t out_bs = L * C; - CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yb = output + n * out_bs; - const __fp16* Wc = weight + c * K; - - if (dilation == 1) { - - size_t i = 0; - for(; i + 7 < L; i += 8){ - float16x8_t acc = vdupq_n_f16(0.0f); - - for (size_t k = 0; k < K; ++k) { - float16x8_t input_vec = vld1q_f16(&Xb[(i + k) * C + c]); - float16x8_t weight_vec = vdupq_n_f16(Wc[k]); - acc = vfmaq_f16(acc, input_vec, weight_vec); - } - vst1q_f16(&Yb[i * C + c], acc); - } - - for (; i < L; ++i) { - float acc = 0.0f; - for (size_t k = 0; k < K; ++k) { - acc += float(Xb[(i + k) * C + c]) * float(Wc[k]); - } - Yb[i * C + c] = (__fp16)acc; - } - } else { - - for (size_t i = 0; i < L; ++i) { - float acc = 0.0f; - - for (size_t k = 0; k < K; ++k) { - size_t idx = i + k * dilation; - acc += float(Xb[idx * C + c]) * float(Wc[k]); - } - - Yb[i * C + c] = (__fp16)acc; - } - } - }); -} -#endif - -void cactus_conv1d_causal_depthwise_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, size_t C, size_t K, size_t dilation) -{ -#ifdef __APPLE__ - if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { - conv1d_causal_depthwise_f16_accelerate(input, weight, output, N, L, C, K, dilation); - return; - } -#endif - - const size_t in_bs = L * C; - const size_t out_bs = L * C; - - CactusThreading::parallel_for_2d(N, C, CactusThreading::Thresholds::ATTENTION, [&](size_t n, size_t c) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yb = output + n * out_bs; - - std::vector wrev(K); - const __fp16* Wc = weight + c * K; - for (size_t k = 0; k < K; ++k) wrev[k] = (float)Wc[K - 1 - k]; - - for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { - const size_t t1 = std::min(t0 + 1, L - 1); - - float32x4_t vacc0 = vdupq_n_f32(0.f); - float32x4_t vacc1 = vdupq_n_f32(0.f); - - size_t k = 0; - for (; k + 8 <= K; k += 8) { - - float x0_0=0, x1_0=0, x2_0=0, x3_0=0; - float x0_1=0, x1_1=0, x2_1=0, x3_1=0; - { - ptrdiff_t a0=(ptrdiff_t)t0-(ptrdiff_t)((k+0)*dilation); - ptrdiff_t a1=(ptrdiff_t)t0-(ptrdiff_t)((k+1)*dilation); - ptrdiff_t a2=(ptrdiff_t)t0-(ptrdiff_t)((k+2)*dilation); - ptrdiff_t a3=(ptrdiff_t)t0-(ptrdiff_t)((k+3)*dilation); - if (a0>=0) x0_0 = (float)Xb[(size_t)a0*C + c]; - if (a1>=0) x1_0 = (float)Xb[(size_t)a1*C + c]; - if (a2>=0) x2_0 = (float)Xb[(size_t)a2*C + c]; - if (a3>=0) x3_0 = (float)Xb[(size_t)a3*C + c]; - - ptrdiff_t b0=(ptrdiff_t)t1-(ptrdiff_t)((k+0)*dilation); - ptrdiff_t b1=(ptrdiff_t)t1-(ptrdiff_t)((k+1)*dilation); - ptrdiff_t b2=(ptrdiff_t)t1-(ptrdiff_t)((k+2)*dilation); - ptrdiff_t b3=(ptrdiff_t)t1-(ptrdiff_t)((k+3)*dilation); - if (b0>=0) x0_1 = (float)Xb[(size_t)b0*C + c]; - if (b1>=0) x1_1 = (float)Xb[(size_t)b1*C + c]; - if (b2>=0) x2_1 = (float)Xb[(size_t)b2*C + c]; - if (b3>=0) x3_1 = (float)Xb[(size_t)b3*C + c]; - } - float32x4_t xv0 = {x0_0,x1_0,x2_0,x3_0}; - float32x4_t yv0 = {x0_1,x1_1,x2_1,x3_1}; - float32x4_t wv0 = {wrev[k+0],wrev[k+1],wrev[k+2],wrev[k+3]}; - vacc0 = vfmaq_f32(vacc0, xv0, wv0); - vacc1 = vfmaq_f32(vacc1, yv0, wv0); - - float a0_0=0, a1_0=0, a2_0=0, a3_0=0; - float a0_1=0, a1_1=0, a2_1=0, a3_1=0; - { - ptrdiff_t a0i=(ptrdiff_t)t0-(ptrdiff_t)((k+4)*dilation); - ptrdiff_t a1i=(ptrdiff_t)t0-(ptrdiff_t)((k+5)*dilation); - ptrdiff_t a2i=(ptrdiff_t)t0-(ptrdiff_t)((k+6)*dilation); - ptrdiff_t a3i=(ptrdiff_t)t0-(ptrdiff_t)((k+7)*dilation); - if (a0i>=0) a0_0 = (float)Xb[(size_t)a0i*C + c]; - if (a1i>=0) a1_0 = (float)Xb[(size_t)a1i*C + c]; - if (a2i>=0) a2_0 = (float)Xb[(size_t)a2i*C + c]; - if (a3i>=0) a3_0 = (float)Xb[(size_t)a3i*C + c]; - - ptrdiff_t b0i=(ptrdiff_t)t1-(ptrdiff_t)((k+4)*dilation); - ptrdiff_t b1i=(ptrdiff_t)t1-(ptrdiff_t)((k+5)*dilation); - ptrdiff_t b2i=(ptrdiff_t)t1-(ptrdiff_t)((k+6)*dilation); - ptrdiff_t b3i=(ptrdiff_t)t1-(ptrdiff_t)((k+7)*dilation); - if (b0i>=0) a0_1 = (float)Xb[(size_t)b0i*C + c]; - if (b1i>=0) a1_1 = (float)Xb[(size_t)b1i*C + c]; - if (b2i>=0) a2_1 = (float)Xb[(size_t)b2i*C + c]; - if (b3i>=0) a3_1 = (float)Xb[(size_t)b3i*C + c]; - } - float32x4_t xv1 = {a0_0,a1_0,a2_0,a3_0}; - float32x4_t yv1 = {a0_1,a1_1,a2_1,a3_1}; - float32x4_t wv1 = {wrev[k+4],wrev[k+5],wrev[k+6],wrev[k+7]}; - vacc0 = vfmaq_f32(vacc0, xv1, wv1); - vacc1 = vfmaq_f32(vacc1, yv1, wv1); - } - - float acc0 = vaddvq_f32(vacc0); - float acc1 = vaddvq_f32(vacc1); - - for (; k < K; ++k) { - ptrdiff_t a=(ptrdiff_t)t0-(ptrdiff_t)(k*dilation); - if (a>=0) acc0 += wrev[k] * (float)Xb[(size_t)a*C + c]; - ptrdiff_t b=(ptrdiff_t)t1-(ptrdiff_t)(k*dilation); - if (b>=0) acc1 += wrev[k] * (float)Xb[(size_t)b*C + c]; - } - - Yb[t0*C + c] = (__fp16)acc0; - if (t0 + 1 < L) Yb[t1*C + c] = (__fp16)acc1; - } - }); -} - -void cactus_conv1d_f16_k3( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t stride -){ - const size_t out_len = ((L - 1) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * 3; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, config, [&](size_t n, size_t oc) { - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * 3); - - for (size_t out_idx = 0; out_idx < out_len; out_idx += 2) { - const size_t out_t0 = out_idx; - const bool have_t1 = (out_idx + 1) < out_len; - const size_t out_t1 = have_t1 ? (out_idx + 1) : out_idx; - - const size_t t0 = out_t0 * stride; - const size_t t1 = have_t1 ? (out_t1 * stride) : t0; - - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - size_t ic = 0; - for (; ic + 16 <= C_in; ic += 16) { - for (size_t u = 0; u < 16; ++u) { - const __fp16* Xc = Xb + (ic + u) * L; - const __fp16* Wc = Woc + (ic + u) * 3; - - const float16x8_t wv = { - Wc[0], Wc[1], Wc[2], (__fp16)0, - Wc[0], Wc[1], Wc[2], (__fp16)0 - }; - - const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; - const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; - const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; - const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; - - const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; - const __fp16 x00 = Xc[t0]; - const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; - - __fp16 x1m = 0, x10 = 0, x1p = 0; - if (have_t1) { - x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; - x10 = Xc[t1]; - x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; - } - - const float16x8_t xv = { - x0m, x00, x0p, (__fp16)0, - x1m, x10, x1p, (__fp16)0 - }; - - const float16x4_t xv0_h = vget_low_f16(xv); - const float16x4_t wv0_h = vget_low_f16(wv); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); - - if (have_t1) { - const float16x4_t xv1_h = vget_high_f16(xv); - const float16x4_t wv1_h = vget_high_f16(wv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); - } - } - } - - for (; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L; - const __fp16* Wc = Woc + ic * 3; - - const float16x8_t wv = { - Wc[0], Wc[1], Wc[2], (__fp16)0, - Wc[0], Wc[1], Wc[2], (__fp16)0 - }; - - const ptrdiff_t tm0 = (ptrdiff_t)t0 - 1; - const ptrdiff_t tp0 = (ptrdiff_t)t0 + 1; - const ptrdiff_t tm1 = (ptrdiff_t)t1 - 1; - const ptrdiff_t tp1 = (ptrdiff_t)t1 + 1; - - const __fp16 x0m = (tm0 >= 0) ? Xc[tm0] : (__fp16)0; - const __fp16 x00 = Xc[t0]; - const __fp16 x0p = (tp0 < (ptrdiff_t)L) ? Xc[tp0] : (__fp16)0; - - __fp16 x1m = 0, x10 = 0, x1p = 0; - if (have_t1) { - x1m = (tm1 >= 0) ? Xc[tm1] : (__fp16)0; - x10 = Xc[t1]; - x1p = (tp1 < (ptrdiff_t)L) ? Xc[tp1] : (__fp16)0; - } - - const float16x8_t xv = { - x0m, x00, x0p, (__fp16)0, - x1m, x10, x1p, (__fp16)0 - }; - - const float16x4_t xv0_h = vget_low_f16(xv); - const float16x4_t wv0_h = vget_low_f16(wv); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(xv0_h), vcvt_f32_f16(wv0_h)); - - if (have_t1) { - const float16x4_t xv1_h = vget_high_f16(xv); - const float16x4_t wv1_h = vget_high_f16(wv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(xv1_h), vcvt_f32_f16(wv1_h)); - } - } - - float32x2_t s0 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); - float sum0 = vget_lane_f32(s0, 0) + vget_lane_f32(s0, 1); - Yoc[out_t0] = (__fp16)sum0; - - if (have_t1) { - float32x2_t s1 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); - float sum1 = vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); - Yoc[out_t1] = (__fp16)sum1; - } - } - }); -} - -#ifdef __APPLE__ -static void conv1d_f16_accelerate( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * K; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d( - N, C_out, config, - [&](size_t n, size_t oc) { - - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * K); - const float b = bias ? (float)bias[oc] : 0.f; - - std::vector out_f32(out_len, b); - std::vector input_f32(L); - std::vector weight_f32(K); - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L; - const __fp16* Wc = Woc + ic * K; - - for (size_t i = 0; i < L; ++i) input_f32[i] = (float)Xc[i]; - for (size_t k = 0; k < K; ++k) weight_f32[k] = (float)Wc[k]; - - if (stride == 1) { - std::vector conv_out(out_len); - vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, - conv_out.data(), 1, out_len, K); - vDSP_vadd(out_f32.data(), 1, conv_out.data(), 1, - out_f32.data(), 1, out_len); - } else { - std::vector full_conv(L - K + 1); - vDSP_conv(input_f32.data(), 1, weight_f32.data(), 1, - full_conv.data(), 1, L - K + 1, K); - for (size_t out_t = 0; out_t < out_len; ++out_t) { - out_f32[out_t] += full_conv[out_t * stride]; - } - } - } - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - Yoc[out_t] = (__fp16)out_f32[out_t]; - } - }); -} -#endif - -static void conv1d_f16_neon( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = C_out * out_len; - - const size_t total_compute = N * C_out * out_len * C_in * K; - CactusThreading::ParallelConfig config = (total_compute < 100000) - ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d( - N, C_out, config, - [&](size_t n, size_t oc) { - - const __fp16* Xb = input + n * in_bs; - __fp16* Yoc = output + n * out_bs + oc * out_len; - const __fp16* Woc = weight + oc * (C_in * K); - const float b = bias ? (float)bias[oc] : 0.f; - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - const size_t t = out_t * stride; - float sum = b; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L + t; - const __fp16* Wc = Woc + ic * K; - - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - size_t k = 0; - - for (; k + 8 <= K; k += 8) { - const float16x8_t xv = vld1q_f16(Xc + k); - const float16x8_t wv = vld1q_f16(Wc + k); - - acc0 = vfmaq_f32(acc0, - vcvt_f32_f16(vget_low_f16(xv)), - vcvt_f32_f16(vget_low_f16(wv))); - acc1 = vfmaq_f32(acc1, - vcvt_f32_f16(vget_high_f16(xv)), - vcvt_f32_f16(vget_high_f16(wv))); - } - - float32x2_t s1 = vadd_f32(vget_low_f32(acc0), vget_high_f32(acc0)); - sum += vget_lane_f32(s1, 0) + vget_lane_f32(s1, 1); - - float32x2_t s2 = vadd_f32(vget_low_f32(acc1), vget_high_f32(acc1)); - sum += vget_lane_f32(s2, 0) + vget_lane_f32(s2, 1); - - for (; k < K; ++k) { - sum += (float)Xc[k] * (float)Wc[k]; - } - } - - Yoc[out_t] = (__fp16)sum; - } - }); -} - -#ifdef __APPLE__ -static void conv1d_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -) { - const size_t out_len = (L - K) / stride + 1; - const size_t col_K = C_in * K; - - std::vector W_f32(C_out * col_K); - for (size_t i = 0; i < C_out * col_K; ++i) - W_f32[i] = static_cast(weight[i]); - - std::vector bias_f32; - if (bias) { - bias_f32.resize(C_out); - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - } - - std::vector col(col_K * out_len); - std::vector Y_f32(C_out * out_len); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * L; - __fp16* Yn = output + n * C_out * out_len; - - if (stride == 1) { - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xn + ic * L; - for (size_t k = 0; k < K; ++k) { - float* dst = col.data() + (ic * K + k) * out_len; - const __fp16* src = Xc + k; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float16x8_t v = vld1q_f16(src + t); - vst1q_f32(dst + t, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst + t + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; t < out_len; ++t) - dst[t] = static_cast(src[t]); - } - } - } else { - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xn + ic * L; - for (size_t k = 0; k < K; ++k) { - float* dst = col.data() + (ic * K + k) * out_len; - for (size_t t = 0; t < out_len; ++t) - dst[t] = static_cast(Xc[t * stride + k]); - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(out_len), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(out_len), - 0.0f, Y_f32.data(), static_cast(out_len)); - - if (bias) { - for (size_t oc = 0; oc < C_out; ++oc) { - float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * out_len; - __fp16* dst = Yn + oc * out_len; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + t), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + t + 4), vdupq_n_f32(b)); - vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; t < out_len; ++t) - dst[t] = static_cast<__fp16>(src[t] + b); - } - } else { - for (size_t oc = 0; oc < C_out; ++oc) { - const float* src = Y_f32.data() + oc * out_len; - __fp16* dst = Yn + oc * out_len; - size_t t = 0; - for (; t + 8 <= out_len; t += 8) { - float32x4_t v0 = vld1q_f32(src + t); - float32x4_t v1 = vld1q_f32(src + t + 4); - vst1q_f16(dst + t, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; t < out_len; ++t) - dst[t] = static_cast<__fp16>(src[t]); - } - } - } -} -#endif - -void cactus_conv1d_f16( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t C_out, - size_t K, - size_t stride -){ -#ifdef __APPLE__ - if (stride > 1 || K < ACCELERATE_K_THRESHOLD) { - conv1d_f16_gemm(input, weight, bias, output, N, L, C_in, C_out, K, stride); - return; - } - if (K >= ACCELERATE_K_THRESHOLD && L >= ACCELERATE_L_THRESHOLD) { - conv1d_f16_accelerate(input, weight, bias, output, N, L, C_in, C_out, K, stride); - return; - } -#endif - conv1d_f16_neon(input, weight, bias, output, N, L, C_in, C_out, K, stride); -} - -inline void conv1d_k7s3_oc8_t4( - const __fp16* Xb, - const __fp16* Wpack, - const __fp16* bias, - __fp16* Yb, - size_t L, - size_t out_len, - size_t C_in, - size_t C_out, - size_t out_t, - size_t oc0 -){ - float32x4_t acc[4][2]; - float32x4_t b0 = vdupq_n_f32(0.f); - float32x4_t b1 = vdupq_n_f32(0.f); - if (bias) { - float16x8_t bv = vld1q_f16(bias + oc0); - b0 = vcvt_f32_f16(vget_low_f16(bv)); - b1 = vcvt_f32_f16(vget_high_f16(bv)); - } - - for (int j = 0; j < 4; ++j) { - acc[j][0] = b0; - acc[j][1] = b1; - } - - const size_t t_base = out_t * 3; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; - const __fp16* Xic = Xb + ic * L + t_base; - - for (int k = 0; k < 7; ++k) { - float16x8_t w_half = vld1q_f16(Wic + k * C_out); - float32x4_t w0 = vcvt_f32_f16(vget_low_f16(w_half)); - float32x4_t w1 = vcvt_f32_f16(vget_high_f16(w_half)); - for (int j = 0; j < 4; ++j) { - float x_val = (float)Xic[j * 3 + k]; - float32x4_t xv = vdupq_n_f32(x_val); - - acc[j][0] = vfmaq_f32(acc[j][0], w0, xv); - acc[j][1] = vfmaq_f32(acc[j][1], w1, xv); - } - } - } - - float tmp0[4], tmp1[4]; - for(int j=0; j<4; ++j) { - vst1q_f32(tmp0, acc[j][0]); - vst1q_f32(tmp1, acc[j][1]); - - for(int i=0; i<4; ++i) { - Yb[(oc0 + i) * out_len + out_t + j] = (__fp16)tmp0[i]; - } - for(int i=0; i<4; ++i) { - Yb[(oc0 + 4 + i) * out_len + out_t + j] = (__fp16)tmp1[i]; - } - } -} - -inline void conv1d_k7s3_oc8_scalar( - const __fp16* Xb, - const __fp16* Wpack, - const __fp16* bias, - __fp16* Yb, - size_t L, - size_t out_len, - size_t C_in, - size_t C_out, - size_t out_t, - size_t oc0 -){ - float32x4_t acc0 = vdupq_n_f32(0.f); - float32x4_t acc1 = vdupq_n_f32(0.f); - - if (bias) { - float16x8_t bv = vld1q_f16(bias + oc0); - acc0 = vcvt_f32_f16(vget_low_f16(bv)); - acc1 = vcvt_f32_f16(vget_high_f16(bv)); - } - - const size_t t_base = out_t * 3; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Wic = Wpack + (ic * 7) * C_out + oc0; - const __fp16* Xic = Xb + ic * L + t_base; - for (int k = 0; k < 7; ++k) { - float x_val = (float)Xic[k]; - float32x4_t xv = vdupq_n_f32(x_val); - - float16x8_t w_half = vld1q_f16(Wic + k * C_out); - acc0 = vfmaq_f32(acc0, vcvt_f32_f16(vget_low_f16(w_half)), xv); - acc1 = vfmaq_f32(acc1, vcvt_f32_f16(vget_high_f16(w_half)), xv); - } - } - - float tmp[8]; - vst1q_f32(tmp, acc0); - vst1q_f32(tmp+4, acc1); - - for(int i=0; i<8; ++i) { - if (oc0 + i < C_out) { - Yb[(oc0 + i) * out_len + out_t] = (__fp16)tmp[i]; - } - } -} - -void cactus_conv1d_f16_k7s3_oc8( - const __fp16* input, - const __fp16* Wpack, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out) -{ - if (L < 7) return; - size_t out_len = (L - 7) / 3 + 1; - size_t num_oc_blocks = (C_out + 7) / 8; - - CactusThreading::parallel_for_2d(N, num_oc_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, [=](size_t n, size_t ob) { - size_t oc0 = ob * 8; - const __fp16* Xb = input + n * (C_in * L); - __fp16* Yb = output + n * (C_out * out_len); - - size_t out_t = 0; - for (; out_t + 4 <= out_len; out_t += 4) { - conv1d_k7s3_oc8_t4(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); - } - - for (; out_t < out_len; ++out_t) { - conv1d_k7s3_oc8_scalar(Xb, Wpack, bias, Yb, L, out_len, C_in, C_out, out_t, oc0); - } - }); -} - - - -void cactus_bilinear_interpolation_f16(const __fp16* input, __fp16* output, size_t src_height, size_t src_width, size_t embed_dim, - size_t dst_height, size_t dst_width, bool align_corners) -{ - float scale_h, scale_w; - if (align_corners) { - scale_h = (src_height > 1 && dst_height > 1) - ? static_cast(src_height - 1) / static_cast(dst_height - 1) - : 0.0f; - scale_w = (src_width > 1 && dst_width > 1) - ? static_cast(src_width - 1) / static_cast(dst_width - 1) - : 0.0f; - } else { - scale_h = (dst_height > 0) - ? static_cast(src_height) / static_cast(dst_height) - : 0.0f; - scale_w = (dst_width > 0) - ? static_cast(src_width) / static_cast(dst_width) - : 0.0f; - } - - for (size_t dst_y = 0; dst_y < dst_height; ++dst_y) { - for (size_t dst_x = 0; dst_x < dst_width; ++dst_x) { - float src_y_float, src_x_float; - if (align_corners) { - src_y_float = static_cast(dst_y) * scale_h; - src_x_float = static_cast(dst_x) * scale_w; - } else { - src_y_float = (static_cast(dst_y) + 0.5f) * scale_h - 0.5f; - src_x_float = (static_cast(dst_x) + 0.5f) * scale_w - 0.5f; - if (src_y_float < 0.0f) src_y_float = 0.0f; - if (src_x_float < 0.0f) src_x_float = 0.0f; - const float max_y = static_cast(src_height) - 1.0f; - const float max_x = static_cast(src_width) - 1.0f; - if (src_y_float > max_y) src_y_float = max_y; - if (src_x_float > max_x) src_x_float = max_x; - } - - int y0 = static_cast(std::floor(src_y_float)); - int x0 = static_cast(std::floor(src_x_float)); - - int y1 = ((y0 + 1) < static_cast(src_height)) ? (y0 + 1) : (static_cast(src_height) - 1); - int x1 = ((x0 + 1) < static_cast(src_width)) ? (x0 + 1) : (static_cast(src_width) - 1); - - float dy = src_y_float - y0; - float dx = src_x_float - x0; - - float w00 = (1.0f - dx) * (1.0f - dy); - float w01 = dx * (1.0f - dy); - float w10 = (1.0f - dx) * dy; - float w11 = dx * dy; - - size_t idx00 = (y0 * static_cast(src_width) + x0) * static_cast(embed_dim); - size_t idx01 = (y0 * static_cast(src_width) + x1) * static_cast(embed_dim); - size_t idx10 = (y1 * static_cast(src_width) + x0) * static_cast(embed_dim); - size_t idx11 = (y1 * static_cast(src_width) + x1) * static_cast(embed_dim); - - size_t out_idx = (dst_y * dst_width + dst_x) * embed_dim; - - size_t d = 0; - for(; d + 8 <= embed_dim; d += 8){ - float16x8_t v00 = vld1q_f16(input + idx00 + d); - float16x8_t v01 = vld1q_f16(input + idx01 + d); - float16x8_t v10 = vld1q_f16(input + idx10 + d); - float16x8_t v11 = vld1q_f16(input + idx11 + d); - - float16x8_t result = vfmaq_f16( - vfmaq_f16( - vfmaq_f16(vdupq_n_f16(0.0f), v00, vdupq_n_f16(w00)), - v01, vdupq_n_f16(w01)), - v10, vdupq_n_f16(w10)); - result = vfmaq_f16(result, v11, vdupq_n_f16(w11)); - - vst1q_f16(output + out_idx + d, result); - } - - for (; d < embed_dim; ++d) { - output[out_idx + d] = - input[idx00 + d] * w00 + - input[idx01 + d] * w01 + - input[idx10 + d] * w10 + - input[idx11 + d] * w11; - } - } - } -} - -void cactus_stft_f16( - const __fp16* input, - const __fp16* weight, - __fp16* output, - size_t N, size_t L, - size_t C_in, size_t /*C_out*/, - size_t K, size_t stride, - size_t num_fft_bins -) { - const size_t out_len = ((L - K) / stride) + 1; - const size_t in_bs = C_in * L; - const size_t out_bs = 2 * num_fft_bins * out_len; - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xb = input + n * in_bs; - - for (size_t bin = 0; bin < num_fft_bins; ++bin) { - const __fp16* Wr = weight + bin * (C_in * K); - const __fp16* Wi = weight + (bin + num_fft_bins) * (C_in * K); - - for (size_t out_t = 0; out_t < out_len; ++out_t) { - const size_t t = out_t * stride; - float sum_real = 0.0f; - float sum_imag = 0.0f; - - for (size_t ic = 0; ic < C_in; ++ic) { - const __fp16* Xc = Xb + ic * L + t; - const __fp16* Wrc = Wr + ic * K; - const __fp16* Wic = Wi + ic * K; - - float32x4_t acc_r0 = vdupq_n_f32(0.f); - float32x4_t acc_r1 = vdupq_n_f32(0.f); - float32x4_t acc_i0 = vdupq_n_f32(0.f); - float32x4_t acc_i1 = vdupq_n_f32(0.f); - - size_t k = 0; - for (; k + 8 <= K; k += 8) { - const float16x8_t xv = vld1q_f16(Xc + k); - const float16x8_t wr = vld1q_f16(Wrc + k); - const float16x8_t wi = vld1q_f16(Wic + k); - - acc_r0 = vfmaq_f32(acc_r0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wr))); - acc_r1 = vfmaq_f32(acc_r1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wr))); - acc_i0 = vfmaq_f32(acc_i0, vcvt_f32_f16(vget_low_f16(xv)), vcvt_f32_f16(vget_low_f16(wi))); - acc_i1 = vfmaq_f32(acc_i1, vcvt_f32_f16(vget_high_f16(xv)), vcvt_f32_f16(vget_high_f16(wi))); - } - - sum_real += vaddvq_f32(acc_r0) + vaddvq_f32(acc_r1); - sum_imag += vaddvq_f32(acc_i0) + vaddvq_f32(acc_i1); - - for (; k < K; ++k) { - float x = (float)Xc[k]; - sum_real += x * (float)Wrc[k]; - sum_imag += x * (float)Wic[k]; - } - } - - output[n * out_bs + bin * out_len + out_t] = (__fp16)sum_real; - output[n * out_bs + (bin + num_fft_bins) * out_len + out_t] = (__fp16)sum_imag; - } - } - } -} - -void cactus_conv1d_same_depthwise_f16_k9( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, size_t L, size_t C) -{ - constexpr int K = 9; - - const size_t bs = L * C; - - CactusThreading::parallel_for_2d( - N, C, CactusThreading::Thresholds::ATTENTION, - [&](size_t n, size_t c) { - - const __fp16* Xb = input + n * bs; - __fp16* Yb = output + n * bs; - - const __fp16* Wc = weight + c * K; - const float b = bias ? (float)bias[c] : 0.0f; - - const float w0 = (float)Wc[0]; - const float w1 = (float)Wc[1]; - const float w2 = (float)Wc[2]; - const float w3 = (float)Wc[3]; - const float w4 = (float)Wc[4]; - const float w5 = (float)Wc[5]; - const float w6 = (float)Wc[6]; - const float w7 = (float)Wc[7]; - const float w8 = (float)Wc[8]; - - const float32x4_t wv0 = {w0, w1, w2, w3}; - const float32x4_t wv1 = {w4, w5, w6, w7}; - - for (size_t t0 = 0; t0 < L; t0 += T_TILE_F16) { - const bool have_t1 = (t0 + 1 < L); - const size_t t1 = have_t1 ? (t0 + 1) : t0; - - float x0_0=0, x1_0=0, x2_0=0, x3_0=0; - float x4_0=0, x5_0=0, x6_0=0, x7_0=0; - float x8_0=0; - - { - const ptrdiff_t i0 = (ptrdiff_t)t0 - 4; - const ptrdiff_t i1 = (ptrdiff_t)t0 - 3; - const ptrdiff_t i2 = (ptrdiff_t)t0 - 2; - const ptrdiff_t i3 = (ptrdiff_t)t0 - 1; - if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_0 = (float)Xb[(size_t)i0 * C + c]; - if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_0 = (float)Xb[(size_t)i1 * C + c]; - if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_0 = (float)Xb[(size_t)i2 * C + c]; - if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_0 = (float)Xb[(size_t)i3 * C + c]; - - const ptrdiff_t j4 = (ptrdiff_t)t0 + 0; - const ptrdiff_t j5 = (ptrdiff_t)t0 + 1; - const ptrdiff_t j6 = (ptrdiff_t)t0 + 2; - const ptrdiff_t j7 = (ptrdiff_t)t0 + 3; - const ptrdiff_t j8 = (ptrdiff_t)t0 + 4; - if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_0 = (float)Xb[(size_t)j4 * C + c]; - if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_0 = (float)Xb[(size_t)j5 * C + c]; - if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_0 = (float)Xb[(size_t)j6 * C + c]; - if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_0 = (float)Xb[(size_t)j7 * C + c]; - if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_0 = (float)Xb[(size_t)j8 * C + c]; - } - - float32x4_t acc0v = vdupq_n_f32(0.f); - const float32x4_t xv0_0 = {x0_0, x1_0, x2_0, x3_0}; - const float32x4_t xv1_0 = {x4_0, x5_0, x6_0, x7_0}; - acc0v = vfmaq_f32(acc0v, xv0_0, wv0); - acc0v = vfmaq_f32(acc0v, xv1_0, wv1); - float acc0 = vaddvq_f32(acc0v) + (w8 * x8_0) + b; - - float acc1 = 0.f; - if (have_t1) { - float x0_1=0, x1_1=0, x2_1=0, x3_1=0; - float x4_1=0, x5_1=0, x6_1=0, x7_1=0; - float x8_1=0; - - { - const ptrdiff_t i0 = (ptrdiff_t)t1 - 4; - const ptrdiff_t i1 = (ptrdiff_t)t1 - 3; - const ptrdiff_t i2 = (ptrdiff_t)t1 - 2; - const ptrdiff_t i3 = (ptrdiff_t)t1 - 1; - if (i0 >= 0 && i0 < (ptrdiff_t)L) x0_1 = (float)Xb[(size_t)i0 * C + c]; - if (i1 >= 0 && i1 < (ptrdiff_t)L) x1_1 = (float)Xb[(size_t)i1 * C + c]; - if (i2 >= 0 && i2 < (ptrdiff_t)L) x2_1 = (float)Xb[(size_t)i2 * C + c]; - if (i3 >= 0 && i3 < (ptrdiff_t)L) x3_1 = (float)Xb[(size_t)i3 * C + c]; - - const ptrdiff_t j4 = (ptrdiff_t)t1 + 0; - const ptrdiff_t j5 = (ptrdiff_t)t1 + 1; - const ptrdiff_t j6 = (ptrdiff_t)t1 + 2; - const ptrdiff_t j7 = (ptrdiff_t)t1 + 3; - const ptrdiff_t j8 = (ptrdiff_t)t1 + 4; - if (j4 >= 0 && j4 < (ptrdiff_t)L) x4_1 = (float)Xb[(size_t)j4 * C + c]; - if (j5 >= 0 && j5 < (ptrdiff_t)L) x5_1 = (float)Xb[(size_t)j5 * C + c]; - if (j6 >= 0 && j6 < (ptrdiff_t)L) x6_1 = (float)Xb[(size_t)j6 * C + c]; - if (j7 >= 0 && j7 < (ptrdiff_t)L) x7_1 = (float)Xb[(size_t)j7 * C + c]; - if (j8 >= 0 && j8 < (ptrdiff_t)L) x8_1 = (float)Xb[(size_t)j8 * C + c]; - } - - float32x4_t acc1v = vdupq_n_f32(0.f); - const float32x4_t xv0_1 = {x0_1, x1_1, x2_1, x3_1}; - const float32x4_t xv1_1 = {x4_1, x5_1, x6_1, x7_1}; - acc1v = vfmaq_f32(acc1v, xv0_1, wv0); - acc1v = vfmaq_f32(acc1v, xv1_1, wv1); - acc1 = vaddvq_f32(acc1v) + (w8 * x8_1) + b; - } - - Yb[t0 * C + c] = (__fp16)acc0; - if (have_t1) Yb[t1 * C + c] = (__fp16)acc1; - } - }); -} - - -void cactus_conv2d_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, size_t H, size_t W, - size_t C_out -){ - if (H + 2 < 3 || W + 2 < 3) return; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - -#ifdef __APPLE__ - { - const size_t spatial = H_out * W_out; - const size_t col_K = C_in * 9; - std::vector W_f32(C_out * col_K); - for (size_t oc = 0; oc < C_out; ++oc) - for (size_t ic = 0; ic < C_in; ++ic) - for (size_t kh = 0; kh < 3; ++kh) - for (size_t kw = 0; kw < 3; ++kw) - W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - - std::vector bias_f32(C_out, 0.0f); - if (bias) - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - - std::vector col(col_K * spatial); - std::vector Y_f32(C_out * spatial); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * H * W; - __fp16* Yn = output + n * C_out * spatial; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - float* dst = col.data() + (ic * 9 + kh * 3 + kw) * spatial; - for (size_t oh = 0; oh < H_out; ++oh) { - ptrdiff_t ih = static_cast(oh) * 2 + static_cast(kh) - 1; - float* dst_row = dst + oh * W_out; - if (ih < 0 || ih >= static_cast(H)) { - memset(dst_row, 0, W_out * sizeof(float)); - continue; - } - const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; - const ptrdiff_t iw_offset = static_cast(kw) - 1; - for (size_t ow = 0; ow < W_out; ++ow) { - ptrdiff_t iw = static_cast(ow) * 2 + iw_offset; - dst_row[ow] = (iw < 0 || iw >= static_cast(W)) - ? 0.0f : static_cast(src_row[iw]); - } - } - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(spatial), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(spatial), - 0.0f, Y_f32.data(), static_cast(spatial)); - - for (size_t oc = 0; oc < C_out; ++oc) { - const float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * spatial; - __fp16* dst = Yn + oc * spatial; - size_t i = 0; - for (; i + 8 <= spatial; i += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; i < spatial; ++i) - dst[i] = static_cast<__fp16>(src[i] + b); - } - } - return; - } -#endif - - auto in_idx = [&](size_t n, size_t ic, size_t ih, size_t iw) -> size_t { - return ((n * C_in + ic) * H + ih) * W + iw; - }; - auto w_idx = [&](size_t oc, size_t ic, size_t kh, size_t kw) -> size_t { - return (((oc * C_in + ic) * 3 + kh) * 3 + kw); - }; - auto out_idx = [&](size_t n, size_t oc, size_t oh, size_t ow) -> size_t { - return ((n * C_out + oc) * H_out + oh) * W_out + ow; - }; - - constexpr size_t OW_VEC = 4; - - const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { - const float b0 = bias ? (float)bias[oc] : 0.0f; - - for (size_t oh = 0; oh < H_out; ++oh) { - const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; - - const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); - - for (size_t ow = 0; ow < W_out; ) { - const size_t rem = W_out - ow; - - const bool do_vec = (rem >= OW_VEC); - const bool w_interior_vec = - do_vec && - (ow >= 1) && - (((ow + (OW_VEC - 1)) * 2 + 1) < W); - - float32x4_t vacc = vdupq_n_f32(b0); - - if (h_interior && w_interior_vec) { - for (size_t ic = 0; ic < C_in; ++ic) { - - const __fp16* row0 = input + in_idx(n, ic, (size_t)(ih0 + 0), 0); - const __fp16* row1 = input + in_idx(n, ic, (size_t)(ih0 + 1), 0); - const __fp16* row2 = input + in_idx(n, ic, (size_t)(ih0 + 2), 0); - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - { - const float w00 = (float)weight[w_idx(oc, ic, 0, 0)]; - const float w01 = (float)weight[w_idx(oc, ic, 0, 1)]; - const float w02 = (float)weight[w_idx(oc, ic, 0, 2)]; - - float x0 = (float)row0[(size_t)(iw_base0 + 0)]; - float x1 = (float)row0[(size_t)(iw_base0 + 2)]; - float x2 = (float)row0[(size_t)(iw_base0 + 4)]; - float x3 = (float)row0[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w00)); - - x0 = (float)row0[(size_t)(iw_base0 + 1)]; - x1 = (float)row0[(size_t)(iw_base0 + 3)]; - x2 = (float)row0[(size_t)(iw_base0 + 5)]; - x3 = (float)row0[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w01)); - - x0 = (float)row0[(size_t)(iw_base0 + 2)]; - x1 = (float)row0[(size_t)(iw_base0 + 4)]; - x2 = (float)row0[(size_t)(iw_base0 + 6)]; - x3 = (float)row0[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w02)); - } - - { - const float w10 = (float)weight[w_idx(oc, ic, 1, 0)]; - const float w11 = (float)weight[w_idx(oc, ic, 1, 1)]; - const float w12 = (float)weight[w_idx(oc, ic, 1, 2)]; - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float x0 = (float)row1[(size_t)(iw_base0 + 0)]; - float x1 = (float)row1[(size_t)(iw_base0 + 2)]; - float x2 = (float)row1[(size_t)(iw_base0 + 4)]; - float x3 = (float)row1[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w10)); - - x0 = (float)row1[(size_t)(iw_base0 + 1)]; - x1 = (float)row1[(size_t)(iw_base0 + 3)]; - x2 = (float)row1[(size_t)(iw_base0 + 5)]; - x3 = (float)row1[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w11)); - - x0 = (float)row1[(size_t)(iw_base0 + 2)]; - x1 = (float)row1[(size_t)(iw_base0 + 4)]; - x2 = (float)row1[(size_t)(iw_base0 + 6)]; - x3 = (float)row1[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w12)); - } - - { - const float w20 = (float)weight[w_idx(oc, ic, 2, 0)]; - const float w21 = (float)weight[w_idx(oc, ic, 2, 1)]; - const float w22 = (float)weight[w_idx(oc, ic, 2, 2)]; - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float x0 = (float)row2[(size_t)(iw_base0 + 0)]; - float x1 = (float)row2[(size_t)(iw_base0 + 2)]; - float x2 = (float)row2[(size_t)(iw_base0 + 4)]; - float x3 = (float)row2[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w20)); - - x0 = (float)row2[(size_t)(iw_base0 + 1)]; - x1 = (float)row2[(size_t)(iw_base0 + 3)]; - x2 = (float)row2[(size_t)(iw_base0 + 5)]; - x3 = (float)row2[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w21)); - - x0 = (float)row2[(size_t)(iw_base0 + 2)]; - x1 = (float)row2[(size_t)(iw_base0 + 4)]; - x2 = (float)row2[(size_t)(iw_base0 + 6)]; - x3 = (float)row2[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0,x1,x2,x3}, vdupq_n_f32(w22)); - } - } - - __fp16* Y = output + out_idx(n, oc, oh, ow); - const float16x4_t yv = vcvt_f16_f32(vacc); - vst1_f16(Y, yv); - - ow += OW_VEC; - continue; - } - - const size_t tile = do_vec ? OW_VEC : 1; - float acc_s[OW_VEC] = {b0, b0, b0, b0}; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; - if (ih < 0 || ih >= (ptrdiff_t)H) continue; - - const __fp16* row = input + in_idx(n, ic, (size_t)ih, 0); - - for (size_t kw = 0; kw < 3; ++kw) { - const float wv = (float)weight[w_idx(oc, ic, kh, kw)]; - for (size_t t = 0; t < tile; ++t) { - const size_t ow_t = ow + t; - const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; - if (iw < 0 || iw >= (ptrdiff_t)W) continue; - acc_s[t] += (float)row[(size_t)iw] * wv; - } - } - } - } - - __fp16* Y = output + out_idx(n, oc, oh, ow); - for (size_t t = 0; t < tile; ++t) { - Y[t] = (__fp16)acc_s[t]; - } - ow += tile; - } - } - }); -} - -void cactus_conv2d_depthwise_f16_k3s2p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C, - size_t H, - size_t W -){ - if (H + 2 < 3 || W + 2 < 3) return; - const size_t H_out = (H - 1) / 2 + 1; - const size_t W_out = (W - 1) / 2 + 1; - - auto in_idx = [&](size_t n, size_t c, size_t ih, size_t iw) -> size_t { - return ((n * C + c) * H + ih) * W + iw; - }; - auto out_idx = [&](size_t n, size_t c, size_t oh, size_t ow) -> size_t { - return ((n * C + c) * H_out + oh) * W_out + ow; - }; - - constexpr size_t OW_VEC = 4; - const size_t total_compute = N * C * H_out * W_out * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C, cfg, [&](size_t n, size_t c) { - const __fp16* Wc = weight + c * 9; - const float w00 = (float)Wc[0], w01 = (float)Wc[1], w02 = (float)Wc[2]; - const float w10 = (float)Wc[3], w11 = (float)Wc[4], w12 = (float)Wc[5]; - const float w20 = (float)Wc[6], w21 = (float)Wc[7], w22 = (float)Wc[8]; - const float b0 = bias ? (float)bias[c] : 0.0f; - - for (size_t oh = 0; oh < H_out; ++oh) { - const ptrdiff_t ih0 = (ptrdiff_t)oh * 2 - 1; - const bool h_interior = (ih0 >= 0) && ((size_t)(ih0 + 2) < H); - - for (size_t ow = 0; ow < W_out; ) { - const size_t rem = W_out - ow; - const bool do_vec = (rem >= OW_VEC); - const bool w_interior_vec = - do_vec && - (ow >= 1) && - (((ow + (OW_VEC - 1)) * 2 + 1) < W); - - if (h_interior && w_interior_vec) { - const __fp16* row0 = input + in_idx(n, c, (size_t)(ih0 + 0), 0); - const __fp16* row1 = input + in_idx(n, c, (size_t)(ih0 + 1), 0); - const __fp16* row2 = input + in_idx(n, c, (size_t)(ih0 + 2), 0); - const ptrdiff_t iw_base0 = (ptrdiff_t)ow * 2 - 1; - - float32x4_t vacc = vdupq_n_f32(b0); - float x0, x1, x2, x3; - - x0 = (float)row0[(size_t)(iw_base0 + 0)]; - x1 = (float)row0[(size_t)(iw_base0 + 2)]; - x2 = (float)row0[(size_t)(iw_base0 + 4)]; - x3 = (float)row0[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w00)); - - x0 = (float)row0[(size_t)(iw_base0 + 1)]; - x1 = (float)row0[(size_t)(iw_base0 + 3)]; - x2 = (float)row0[(size_t)(iw_base0 + 5)]; - x3 = (float)row0[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w01)); - - x0 = (float)row0[(size_t)(iw_base0 + 2)]; - x1 = (float)row0[(size_t)(iw_base0 + 4)]; - x2 = (float)row0[(size_t)(iw_base0 + 6)]; - x3 = (float)row0[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w02)); - - x0 = (float)row1[(size_t)(iw_base0 + 0)]; - x1 = (float)row1[(size_t)(iw_base0 + 2)]; - x2 = (float)row1[(size_t)(iw_base0 + 4)]; - x3 = (float)row1[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w10)); - - x0 = (float)row1[(size_t)(iw_base0 + 1)]; - x1 = (float)row1[(size_t)(iw_base0 + 3)]; - x2 = (float)row1[(size_t)(iw_base0 + 5)]; - x3 = (float)row1[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w11)); - - x0 = (float)row1[(size_t)(iw_base0 + 2)]; - x1 = (float)row1[(size_t)(iw_base0 + 4)]; - x2 = (float)row1[(size_t)(iw_base0 + 6)]; - x3 = (float)row1[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w12)); - - x0 = (float)row2[(size_t)(iw_base0 + 0)]; - x1 = (float)row2[(size_t)(iw_base0 + 2)]; - x2 = (float)row2[(size_t)(iw_base0 + 4)]; - x3 = (float)row2[(size_t)(iw_base0 + 6)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w20)); - - x0 = (float)row2[(size_t)(iw_base0 + 1)]; - x1 = (float)row2[(size_t)(iw_base0 + 3)]; - x2 = (float)row2[(size_t)(iw_base0 + 5)]; - x3 = (float)row2[(size_t)(iw_base0 + 7)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w21)); - - x0 = (float)row2[(size_t)(iw_base0 + 2)]; - x1 = (float)row2[(size_t)(iw_base0 + 4)]; - x2 = (float)row2[(size_t)(iw_base0 + 6)]; - x3 = (float)row2[(size_t)(iw_base0 + 8)]; - vacc = vfmaq_f32(vacc, (float32x4_t){x0, x1, x2, x3}, vdupq_n_f32(w22)); - - __fp16* Y = output + out_idx(n, c, oh, ow); - vst1_f16(Y, vcvt_f16_f32(vacc)); - ow += OW_VEC; - continue; - } - - const size_t tile = do_vec ? OW_VEC : 1; - float acc_s[OW_VEC] = {b0, b0, b0, b0}; - for (size_t t = 0; t < tile; ++t) { - const size_t ow_t = ow + t; - for (size_t kh = 0; kh < 3; ++kh) { - const ptrdiff_t ih = ih0 + (ptrdiff_t)kh; - if (ih < 0 || ih >= (ptrdiff_t)H) continue; - - const __fp16* row = input + in_idx(n, c, (size_t)ih, 0); - for (size_t kw = 0; kw < 3; ++kw) { - const ptrdiff_t iw = (ptrdiff_t)ow_t * 2 - 1 + (ptrdiff_t)kw; - if (iw < 0 || iw >= (ptrdiff_t)W) continue; - const float wv = (float)Wc[kh * 3 + kw]; - acc_s[t] += (float)row[(size_t)iw] * wv; - } - } - } - - __fp16* Y = output + out_idx(n, c, oh, ow); - for (size_t t = 0; t < tile; ++t) { - Y[t] = (__fp16)acc_s[t]; - } - ow += tile; - } - } - }); -} - -void cactus_conv2d_pointwise_f16_1x1_nchw_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, - size_t H, - size_t W, - size_t C_out -){ - const size_t HW = H * W; - if (HW == 0 || N == 0) return; - - CactusThreading::parallel_for( - N, - CactusThreading::Thresholds::ATTENTION, - [&](size_t n_start, size_t n_end) { - std::vector<__fp16> packed_in(HW * C_in); - std::vector<__fp16> packed_out(HW * C_out); - - for (size_t n = n_start; n < n_end; ++n) { - const __fp16* Xn = input + n * C_in * HW; - __fp16* Yn = output + n * C_out * HW; - - for (size_t hw = 0; hw < HW; ++hw) { - const size_t h = hw / W; - const size_t w = hw - h * W; - __fp16* row = packed_in.data() + hw * C_in; - for (size_t ic = 0; ic < C_in; ++ic) { - row[ic] = Xn[(ic * H + h) * W + w]; - } - } - - cactus_matmul_f16( - packed_in.data(), - weight, - packed_out.data(), - HW, - C_in, - C_out - ); - - for (size_t hw = 0; hw < HW; ++hw) { - const size_t h = hw / W; - const size_t w = hw - h * W; - const __fp16* row = packed_out.data() + hw * C_out; - for (size_t oc = 0; oc < C_out; ++oc) { - float outv = (float)row[oc]; - if (bias) outv += (float)bias[oc]; - Yn[(oc * H + h) * W + w] = (__fp16)outv; - } - } - } - }); -} - -void cactus_conv1d_pointwise_f16_gemm( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t L, - size_t C_in, - size_t C_out -){ - const size_t M = N * L; - if (M == 0) return; - - cactus_matmul_f16(input, weight, output, M, C_in, C_out); - - if (bias) { - for (size_t m = 0; m < M; ++m) { - __fp16* row = output + m * C_out; - for (size_t oc = 0; oc < C_out; ++oc) { - row[oc] = (__fp16)((float)row[oc] + (float)bias[oc]); - } - } - } -} - -void cactus_conv2d_f16_k3s1p1_nchw( - const __fp16* input, - const __fp16* weight, - const __fp16* bias, - __fp16* output, - size_t N, - size_t C_in, size_t H, size_t W, - size_t C_out -) { - const size_t H_out = H; - const size_t W_out = W; - -#ifdef __APPLE__ - const size_t col_K = C_in * 9; - std::vector W_f32(C_out * col_K); - for (size_t oc = 0; oc < C_out; ++oc) { - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - W_f32[oc * col_K + ic * 9 + kh * 3 + kw] = - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - } - } - } - } - - std::vector bias_f32(C_out, 0.0f); - if (bias) { - for (size_t i = 0; i < C_out; ++i) - bias_f32[i] = static_cast(bias[i]); - } - - std::vector col(col_K * H_out * W_out); - std::vector Y_f32(C_out * H_out * W_out); - - for (size_t n = 0; n < N; ++n) { - const __fp16* Xn = input + n * C_in * H * W; - __fp16* Yn = output + n * C_out * H_out * W_out; - - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - float* dst = col.data() + (ic * 9 + kh * 3 + kw) * H_out * W_out; - for (size_t oh = 0; oh < H_out; ++oh) { - ptrdiff_t ih = static_cast(oh) + static_cast(kh) - 1; - float* dst_row = dst + oh * W_out; - if (ih < 0 || ih >= static_cast(H)) { - memset(dst_row, 0, W_out * sizeof(float)); - continue; - } - const __fp16* src_row = Xn + ic * H * W + static_cast(ih) * W; - const ptrdiff_t iw_offset = static_cast(kw) - 1; - size_t ow_start = 0, ow_end = W_out; - if (iw_offset < 0) { dst_row[0] = 0.0f; ow_start = 1; } - if (iw_offset > 0) { dst_row[W_out - 1] = 0.0f; ow_end = W_out - 1; } - size_t ow = ow_start; - for (; ow + 8 <= ow_end; ow += 8) { - float16x8_t v = vld1q_f16(src_row + ow + iw_offset); - vst1q_f32(dst_row + ow, vcvt_f32_f16(vget_low_f16(v))); - vst1q_f32(dst_row + ow + 4, vcvt_f32_f16(vget_high_f16(v))); - } - for (; ow < ow_end; ++ow) - dst_row[ow] = static_cast(src_row[ow + iw_offset]); - } - } - } - } - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - static_cast(C_out), static_cast(H_out * W_out), static_cast(col_K), - 1.0f, W_f32.data(), static_cast(col_K), - col.data(), static_cast(H_out * W_out), - 0.0f, Y_f32.data(), static_cast(H_out * W_out)); - - for (size_t oc = 0; oc < C_out; ++oc) { - float b = bias_f32[oc]; - const float* src = Y_f32.data() + oc * H_out * W_out; - __fp16* dst = Yn + oc * H_out * W_out; - size_t i = 0; - for (; i + 8 <= H_out * W_out; i += 8) { - float32x4_t v0 = vaddq_f32(vld1q_f32(src + i), vdupq_n_f32(b)); - float32x4_t v1 = vaddq_f32(vld1q_f32(src + i + 4), vdupq_n_f32(b)); - vst1q_f16(dst + i, vcombine_f16(vcvt_f16_f32(v0), vcvt_f16_f32(v1))); - } - for (; i < H_out * W_out; ++i) - dst[i] = static_cast<__fp16>(src[i] + b); - } - } - -#else - const size_t total_compute = N * C_out * H_out * W_out * C_in * 9; - CactusThreading::ParallelConfig cfg = - (total_compute < 100000) ? CactusThreading::ParallelConfig{SIZE_MAX, SIZE_MAX} - : CactusThreading::Thresholds::ATTENTION; - - CactusThreading::parallel_for_2d(N, C_out, cfg, [&](size_t n, size_t oc) { - const float b0 = bias ? static_cast(bias[oc]) : 0.0f; - for (size_t oh = 0; oh < H_out; ++oh) { - for (size_t ow = 0; ow < W_out; ++ow) { - float acc = b0; - for (size_t ic = 0; ic < C_in; ++ic) { - for (size_t kh = 0; kh < 3; ++kh) { - for (size_t kw = 0; kw < 3; ++kw) { - const ptrdiff_t ih = static_cast(oh) + kh - 1; - const ptrdiff_t iw = static_cast(ow) + kw - 1; - if (ih >= 0 && ih < static_cast(H) && - iw >= 0 && iw < static_cast(W)) { - acc += static_cast(input[((n * C_in + ic) * H + ih) * W + iw]) * - static_cast(weight[((oc * C_in + ic) * 3 + kh) * 3 + kw]); - } - } - } - } - output[((n * C_out + oc) * H_out + oh) * W_out + ow] = static_cast<__fp16>(acc); - } - } - }); -#endif -} - -void cactus_maxpool1d_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t channels, - size_t input_length, - size_t kernel_size, - size_t stride -) { - const size_t output_length = (input_length - kernel_size) / stride + 1; - - CactusThreading::parallel_for(batch_size * channels, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t start, size_t end) { - for (size_t bc = start; bc < end; ++bc) { - const size_t b = bc / channels; - const size_t c = bc % channels; - - const __fp16* src = input + b * channels * input_length + c * input_length; - __fp16* dst = output + b * channels * output_length + c * output_length; - - for (size_t i = 0; i < output_length; ++i) { - const size_t in_start = i * stride; - float max_val = -std::numeric_limits::infinity(); - for (size_t k = 0; k < kernel_size; ++k) { - float val = static_cast(src[in_start + k]); - if (val > max_val) max_val = val; - } - dst[i] = static_cast<__fp16>(max_val); - } - } - }); -} diff --git a/cactus/kernel/kernel_deltanet.cpp b/cactus/kernel/kernel_deltanet.cpp deleted file mode 100644 index 4972bc49b..000000000 --- a/cactus/kernel/kernel_deltanet.cpp +++ /dev/null @@ -1,677 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" - -#include -#include -#include -#include -#include - -namespace { - -inline float safe_exp_gate(float gate_log) { - const float clamped = std::max(-20.0f, std::min(6.0f, gate_log)); - return std::exp(clamped); -} - -inline void fold_state_scale(std::vector& state, float factor) { - const float32x4_t f4 = vdupq_n_f32(factor); - size_t i = 0; - for (; i + 8 <= state.size(); i += 8) { - vst1q_f32(&state[i], vmulq_f32(vld1q_f32(&state[i]), f4)); - vst1q_f32(&state[i + 4], vmulq_f32(vld1q_f32(&state[i + 4]), f4)); - } - for (; i < state.size(); ++i) state[i] *= factor; -} - -struct GatedDeltaChunkScratch { - std::vector state; - std::vector m_chunk; - std::vector q_chunk; - std::vector k_chunk; - std::vector v_chunk; - std::vector g_chunk; - std::vector beta_chunk; - std::vector n0_proj; - std::vector delta_chunk; - std::vector gram; - std::vector p_prev; - std::vector p_curr; - std::vector inv_p_curr; - std::vector coeff; - std::vector out_acc; - - void ensure(size_t k_dim, size_t v_dim, size_t c_max) { - const size_t kv = k_dim * v_dim; - if (state.size() < kv) state.resize(kv); - if (m_chunk.size() < kv) m_chunk.resize(kv); - if (q_chunk.size() < c_max * k_dim) q_chunk.resize(c_max * k_dim); - if (k_chunk.size() < c_max * k_dim) k_chunk.resize(c_max * k_dim); - if (v_chunk.size() < c_max * v_dim) v_chunk.resize(c_max * v_dim); - if (g_chunk.size() < c_max) g_chunk.resize(c_max); - if (beta_chunk.size() < c_max) beta_chunk.resize(c_max); - if (n0_proj.size() < c_max * v_dim) n0_proj.resize(c_max * v_dim); - if (delta_chunk.size() < c_max * v_dim) delta_chunk.resize(c_max * v_dim); - if (gram.size() < c_max * c_max) gram.resize(c_max * c_max); - if (p_prev.size() < c_max) p_prev.resize(c_max); - if (p_curr.size() < c_max) p_curr.resize(c_max); - if (inv_p_curr.size() < c_max) inv_p_curr.resize(c_max); - if (coeff.size() < c_max * c_max) coeff.resize(c_max * c_max); - if (out_acc.size() < v_dim) out_acc.resize(v_dim); - } -}; - -thread_local GatedDeltaChunkScratch g_gated_deltanet_chunk_scratch; - -inline size_t tuned_gated_deltanet_chunk_size(size_t requested_chunk, size_t k_dim, size_t v_dim) { - size_t chunk = requested_chunk == 0 ? 64 : requested_chunk; - - const char* env_chunk = std::getenv("CACTUS_GATED_DELTANET_CHUNK_SIZE"); - if (env_chunk != nullptr) { - long parsed = std::strtol(env_chunk, nullptr, 10); - if (parsed > 1) { - chunk = static_cast(parsed); - } - } else { - (void)k_dim; - (void)v_dim; - chunk = std::min(chunk, 16); - } - - return std::max(2, chunk); -} - -void gated_deltanet_step( - const __fp16* q_ptr, - const __fp16* k_ptr, - const __fp16* v_ptr, - float gate_log, - float beta, - float scale, - size_t k_dim, - size_t v_dim, - std::vector& state, - float& state_scale, - std::vector& proj, - std::vector& delta, - __fp16* out_ptr) { - const float gate_log_safe = std::isfinite(gate_log) ? gate_log : -20.0f; - const float beta_safe = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; - - if (proj.size() != v_dim) { - proj.assign(v_dim, 0.0f); - } - if (delta.size() != v_dim) { - delta.assign(v_dim, 0.0f); - } - - const float prev_scale = state_scale; - std::fill(proj.begin(), proj.end(), 0.0f); - - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); - const float k_val = static_cast(k_ptr[kd]); - const float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t k4 = vdupq_n_f32(k_val); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t p = vld1q_f32(proj.data() + vd); - float32x4_t s = vld1q_f32(state_row + vd); - p = vfmaq_f32(p, s, k4); - vst1q_f32(proj.data() + vd, p); - } - for (; vd < v_dim; ++vd) { - proj[vd] += state_row[vd] * k_val; - } - } - { - size_t vd = 0; - const float32x4_t scale4 = vdupq_n_f32(prev_scale); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t p = vld1q_f32(proj.data() + vd); - p = vmulq_f32(p, scale4); - vst1q_f32(proj.data() + vd, p); - } - for (; vd < v_dim; ++vd) { - proj[vd] *= prev_scale; - } - } - - for (size_t vd = 0; vd < v_dim; ++vd) { - const float v_val = static_cast(v_ptr[vd]); - delta[vd] = (v_val - proj[vd]) * beta_safe; - } - - state_scale = prev_scale * safe_exp_gate(gate_log_safe); - if (!std::isfinite(state_scale) || - std::fabs(state_scale) < 1e-8f || - std::fabs(state_scale) > 1e8f) { - fold_state_scale(state, state_scale); - state_scale = 1.0f; - } - const float inv_scale = 1.0f / state_scale; - - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 1, 3); - const float k_scaled = static_cast(k_ptr[kd]) * inv_scale; - float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t k4 = vdupq_n_f32(k_scaled); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t s = vld1q_f32(state_row + vd); - float32x4_t d = vld1q_f32(delta.data() + vd); - s = vfmaq_f32(s, d, k4); - vst1q_f32(state_row + vd, s); - } - for (; vd < v_dim; ++vd) { - state_row[vd] += k_scaled * delta[vd]; - } - for (size_t vd = 0; vd < v_dim; ++vd) { - if (!std::isfinite(state_row[vd])) { - state_row[vd] = 0.0f; - } - } - } - - std::fill(proj.begin(), proj.end(), 0.0f); - for (size_t kd = 0; kd < k_dim; ++kd) { - if (kd + 2 < k_dim) __builtin_prefetch(state.data() + (kd + 2) * v_dim, 0, 3); - const float q_val = static_cast(q_ptr[kd]); - const float* state_row = state.data() + kd * v_dim; - size_t vd = 0; - const float32x4_t q4 = vdupq_n_f32(q_val); - for (; vd + 4 <= v_dim; vd += 4) { - float32x4_t o = vld1q_f32(proj.data() + vd); - float32x4_t s = vld1q_f32(state_row + vd); - o = vfmaq_f32(o, s, q4); - vst1q_f32(proj.data() + vd, o); - } - for (; vd < v_dim; ++vd) { - proj[vd] += state_row[vd] * q_val; - } - } - - const float out_scale = state_scale * scale; - for (size_t vd = 0; vd < v_dim; ++vd) { - float acc = proj[vd]; - if (!std::isfinite(acc)) { - acc = 0.0f; - } - out_ptr[vd] = static_cast<__fp16>(acc * out_scale); - } -} - -void gated_deltanet_prefill_old_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale) { - const size_t out_seq = T + K; - const size_t qk_repeat = Hv / Hq; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - std::vector state(K * V); - std::vector proj(V); - std::vector delta(V); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); - state[kd * V + vd] = static_cast(s_data[s_idx]); - } - } - - float state_scale = 1.0f; - for (size_t t = 0; t < T; ++t) { - const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); - const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); - const size_t gb_idx = ((batch * T + t) * Hv + v_head); - - gated_deltanet_step( - q_data + qkv_k_base, - k_data + qkv_k_base, - v_data + qkv_v_base, - static_cast(g_data[gb_idx]), - static_cast(b_data[gb_idx]), - scale, - K, - V, - state, - state_scale, - proj, - delta, - out + (((batch * out_seq + t) * Hv + v_head) * V)); - } - - fold_state_scale(state, state_scale); - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V + vd); - out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); - } - } - } - }); -} - -void gated_deltanet_prefill_chunked_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t chunk_size, - float scale) { - const size_t out_seq = T + K; - const size_t qk_repeat = Hv / Hq; - constexpr float kMinAbs = 1e-12f; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - auto& ws = g_gated_deltanet_chunk_scratch; - ws.ensure(K, V, chunk_size); - - float* state = ws.state.data(); - float* m_chunk = ws.m_chunk.data(); - float* q_chunk = ws.q_chunk.data(); - float* k_chunk = ws.k_chunk.data(); - float* v_chunk = ws.v_chunk.data(); - float* g_chunk = ws.g_chunk.data(); - float* beta_chunk = ws.beta_chunk.data(); - float* n0_proj = ws.n0_proj.data(); - float* delta_chunk = ws.delta_chunk.data(); - float* gram = ws.gram.data(); - float* p_prev = ws.p_prev.data(); - float* p_curr = ws.p_curr.data(); - float* inv_p_curr = ws.inv_p_curr.data(); - float* coeff = ws.coeff.data(); - float* out_acc = ws.out_acc.data(); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V); - cactus_fp16_to_fp32(s_data + s_idx, state + kd * V, V); - } - - for (size_t t0 = 0; t0 < T; t0 += chunk_size) { - const size_t C = std::min(chunk_size, T - t0); - const size_t CV = C * V; - const size_t CC = C * C; - - std::fill(n0_proj, n0_proj + CV, 0.0f); - std::fill(delta_chunk, delta_chunk + CV, 0.0f); - std::fill(gram, gram + CC, 0.0f); - std::fill(coeff, coeff + CC, 0.0f); - std::fill(m_chunk, m_chunk + (K * V), 0.0f); - - // Fused fp16→fp32 conversion + n0_proj: k_chunk stays in L1 - for (size_t ct = 0; ct < C; ++ct) { - const size_t t = t0 + ct; - const size_t qkv_k_base = (((batch * T + t) * Hq + qk_head) * K); - const size_t qkv_v_base = (((batch * T + t) * Hv + v_head) * V); - const size_t gb_idx = ((batch * T + t) * Hv + v_head); - - // Inline fp16→fp32 for q - { - const __fp16* src = q_data + qkv_k_base; - float* dst = q_chunk + ct * K; - size_t i = 0; - for (; i + 8 <= K; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < K; ++i) dst[i] = static_cast(src[i]); - } - - // Inline fp16→fp32 for k - { - const __fp16* src = k_data + qkv_k_base; - float* dst = k_chunk + ct * K; - size_t i = 0; - for (; i + 8 <= K; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < K; ++i) dst[i] = static_cast(src[i]); - } - - // Inline fp16→fp32 for v - { - const __fp16* src = v_data + qkv_v_base; - float* dst = v_chunk + ct * V; - size_t i = 0; - for (; i + 8 <= V; i += 8) { - float16x8_t in = vld1q_f16(src + i); - vst1q_f32(dst + i, vcvt_f32_f16(vget_low_f16(in))); - vst1q_f32(dst + i + 4, vcvt_f32_f16(vget_high_f16(in))); - } - for (; i < V; ++i) dst[i] = static_cast(src[i]); - } - - const float gate_log = static_cast(g_data[gb_idx]); - const float beta = static_cast(b_data[gb_idx]); - g_chunk[ct] = safe_exp_gate(std::isfinite(gate_log) ? gate_log : -20.0f); - beta_chunk[ct] = std::isfinite(beta) ? std::min(1.0f, std::max(0.0f, beta)) : 0.0f; - - // n0_proj fused here so k_chunk[ct] is still in L1 - const float* k_row = k_chunk + ct * K; - float* p_row = n0_proj + ct * V; - for (size_t kd = 0; kd < K; ++kd) { - const float kval = k_row[kd]; - const float* s_row = state + kd * V; - size_t vd = 0; - const float32x4_t kv4 = vdupq_n_f32(kval); - for (; vd + 8 <= V; vd += 8) { - float32x4_t p4 = vld1q_f32(p_row + vd); - const float32x4_t slo = vld1q_f32(s_row + vd); - const float32x4_t shi = vld1q_f32(s_row + vd + 4); - p4 = vfmaq_f32(p4, slo, kv4); - vst1q_f32(p_row + vd, p4); - p4 = vld1q_f32(p_row + vd + 4); - p4 = vfmaq_f32(p4, shi, kv4); - vst1q_f32(p_row + vd + 4, p4); - } - for (; vd < V; ++vd) { - p_row[vd] += kval * s_row[vd]; - } - } - } - - for (size_t t = 0; t < C; ++t) { - const float* kt = k_chunk + t * K; - for (size_t j = 0; j < t; ++j) { - const float* kj = k_chunk + j * K; - float dot = 0.0f; - size_t kd = 0; - float32x4_t acc4 = vdupq_n_f32(0.0f); - for (; kd + 8 <= K; kd += 8) { - const float32x4_t al = vld1q_f32(kt + kd); - const float32x4_t ah = vld1q_f32(kt + kd + 4); - const float32x4_t bl = vld1q_f32(kj + kd); - const float32x4_t bh = vld1q_f32(kj + kd + 4); - acc4 = vfmaq_f32(acc4, al, bl); - acc4 = vfmaq_f32(acc4, ah, bh); - } - dot += vaddvq_f32(acc4); - for (; kd < K; ++kd) { - dot += kt[kd] * kj[kd]; - } - gram[t * C + j] = dot; - } - } - - float running_p = 1.0f; - for (size_t t = 0; t < C; ++t) { - p_prev[t] = running_p; - running_p *= g_chunk[t]; - p_curr[t] = running_p; - inv_p_curr[t] = (std::fabs(running_p) > kMinAbs) ? (1.0f / running_p) : 0.0f; - } - - for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; - float* coeff_row = coeff + t * C; - for (size_t j = 0; j < t; ++j) { - const float inv_pc = inv_p_curr[j]; - coeff_row[j] = inv_pc == 0.0f ? 0.0f : (pp * inv_pc * gram[t * C + j]); - } - } - - for (size_t t = 0; t < C; ++t) { - const float pp = p_prev[t]; - const float bt = beta_chunk[t]; - const float* n0_row = n0_proj + t * V; - const float* v_row = v_chunk + t * V; - float* d_row = delta_chunk + t * V; - - size_t vd = 0; - const float32x4_t pp4 = vdupq_n_f32(pp); - const float32x4_t bt4 = vdupq_n_f32(bt); - for (; vd + 8 <= V; vd += 8) { - float32x4_t acc_lo = vmulq_f32(vld1q_f32(n0_row + vd), pp4); - float32x4_t acc_hi = vmulq_f32(vld1q_f32(n0_row + vd + 4), pp4); - for (size_t j = 0; j < t; ++j) { - const float c = coeff[t * C + j]; - const float* dj = delta_chunk + j * V + vd; - acc_lo = vmlaq_n_f32(acc_lo, vld1q_f32(dj), c); - acc_hi = vmlaq_n_f32(acc_hi, vld1q_f32(dj + 4), c); - } - vst1q_f32(d_row + vd, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd), acc_lo), bt4)); - vst1q_f32(d_row + vd + 4, vmulq_f32(vsubq_f32(vld1q_f32(v_row + vd + 4), acc_hi), bt4)); - } - for (; vd < V; ++vd) { - float acc = pp * n0_row[vd]; - for (size_t j = 0; j < t; ++j) { - acc += coeff[t * C + j] * delta_chunk[j * V + vd]; - } - d_row[vd] = bt * (v_row[vd] - acc); - } - } - - running_p = 1.0f; - for (size_t t = 0; t < C; ++t) { - const float gt = g_chunk[t]; - const float* k_row = k_chunk + t * K; - const float* q_row = q_chunk + t * K; - const float* d_row = delta_chunk + t * V; - - running_p *= gt; - const float p_run = running_p; - - for (size_t kd = 0; kd < K; ++kd) { - const float kval = k_row[kd]; - float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t gt4 = vdupq_n_f32(gt); - const float32x4_t kv4 = vdupq_n_f32(kval); - for (; vd + 8 <= V; vd += 8) { - float32x4_t ml = vld1q_f32(m_row + vd); - float32x4_t mh = vld1q_f32(m_row + vd + 4); - const float32x4_t dl = vld1q_f32(d_row + vd); - const float32x4_t dh = vld1q_f32(d_row + vd + 4); - ml = vmulq_f32(ml, gt4); - mh = vmulq_f32(mh, gt4); - ml = vfmaq_f32(ml, dl, kv4); - mh = vfmaq_f32(mh, dh, kv4); - vst1q_f32(m_row + vd, ml); - vst1q_f32(m_row + vd + 4, mh); - } - for (; vd < V; ++vd) { - m_row[vd] = gt * m_row[vd] + kval * d_row[vd]; - } - } - - const size_t out_base = (((batch * out_seq + (t0 + t)) * Hv + v_head) * V); - std::fill(out_acc, out_acc + V, 0.0f); - for (size_t kd = 0; kd < K; ++kd) { - const float qval = q_row[kd]; - const float* s_row = state + kd * V; - const float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t q4 = vdupq_n_f32(qval); - const float32x4_t p4 = vdupq_n_f32(p_run); - for (; vd + 8 <= V; vd += 8) { - float32x4_t acc4 = vld1q_f32(out_acc + vd); - float32x4_t acc4h = vld1q_f32(out_acc + vd + 4); - const float32x4_t sl = vld1q_f32(s_row + vd); - const float32x4_t sh = vld1q_f32(s_row + vd + 4); - const float32x4_t ml = vld1q_f32(m_row + vd); - const float32x4_t mh = vld1q_f32(m_row + vd + 4); - const float32x4_t nl = vfmaq_f32(ml, sl, p4); - const float32x4_t nh = vfmaq_f32(mh, sh, p4); - acc4 = vfmaq_f32(acc4, nl, q4); - acc4h = vfmaq_f32(acc4h, nh, q4); - vst1q_f32(out_acc + vd, acc4); - vst1q_f32(out_acc + vd + 4, acc4h); - } - for (; vd < V; ++vd) { - out_acc[vd] += qval * (p_run * s_row[vd] + m_row[vd]); - } - } - for (size_t vd = 0; vd < V; ++vd) { - out[out_base + vd] = static_cast<__fp16>(out_acc[vd] * scale); - } - } - - const float p_end = running_p; - for (size_t kd = 0; kd < K; ++kd) { - float* s_row = state + kd * V; - const float* m_row = m_chunk + kd * V; - size_t vd = 0; - const float32x4_t p4 = vdupq_n_f32(p_end); - for (; vd + 8 <= V; vd += 8) { - float32x4_t sl = vld1q_f32(s_row + vd); - float32x4_t sh = vld1q_f32(s_row + vd + 4); - const float32x4_t ml = vld1q_f32(m_row + vd); - const float32x4_t mh = vld1q_f32(m_row + vd + 4); - sl = vfmaq_f32(ml, sl, p4); - sh = vfmaq_f32(mh, sh, p4); - vst1q_f32(s_row + vd, sl); - vst1q_f32(s_row + vd + 4, sh); - } - for (; vd < V; ++vd) { - s_row[vd] = p_end * s_row[vd] + m_row[vd]; - } - } - } - - for (size_t kd = 0; kd < K; ++kd) { - const size_t out_idx = (((batch * out_seq + (T + kd)) * Hv + v_head) * V); - cactus_fp32_to_fp16(state + kd * V, out + out_idx, V); - } - } - }); -} - -} // namespace - -void cactus_gated_deltanet_decode_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - float scale) { - const size_t qk_repeat = Hv / Hq; - const size_t out_seq = 1 + K; - - CactusThreading::parallel_for(B * Hv, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t bh_start, size_t bh_end) { - std::vector state(K * V); - std::vector proj(V); - std::vector delta(V); - - for (size_t bh = bh_start; bh < bh_end; ++bh) { - const size_t batch = bh / Hv; - const size_t v_head = bh % Hv; - const size_t qk_head = v_head / qk_repeat; - - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t s_idx = (((batch * K + kd) * Hv + v_head) * V + vd); - state[kd * V + vd] = static_cast(s_data[s_idx]); - } - } - - float state_scale = 1.0f; - const size_t qk_base = ((batch * Hq + qk_head) * K); - const size_t v_base = ((batch * Hv + v_head) * V); - const float gate_log = static_cast(g_data[batch * Hv + v_head]); - const float beta = static_cast(b_data[batch * Hv + v_head]); - - gated_deltanet_step( - q_data + qk_base, - k_data + qk_base, - v_data + v_base, - gate_log, - beta, - scale, - K, - V, - state, - state_scale, - proj, - delta, - out + (((batch * out_seq) * Hv + v_head) * V)); - - fold_state_scale(state, state_scale); - for (size_t kd = 0; kd < K; ++kd) { - for (size_t vd = 0; vd < V; ++vd) { - const size_t out_idx = (((batch * out_seq + (1 + kd)) * Hv + v_head) * V + vd); - out[out_idx] = static_cast<__fp16>(state[kd * V + vd]); - } - } - } - }); -} - -void cactus_gated_deltanet_prefill_f16( - const __fp16* q_data, - const __fp16* k_data, - const __fp16* v_data, - const __fp16* g_data, - const __fp16* b_data, - const __fp16* s_data, - __fp16* out, - size_t B, - size_t T, - size_t Hq, - size_t Hv, - size_t K, - size_t V, - size_t requested_chunk_size, - float scale) { - const char* force_old = std::getenv("CACTUS_GATED_DELTANET_PREFILL_OLD"); - if (force_old != nullptr && std::atoi(force_old) != 0) { - gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, scale); - return; - } - - if (requested_chunk_size <= 1) { - gated_deltanet_prefill_old_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, scale); - return; - } - - const size_t chunk_size = tuned_gated_deltanet_chunk_size(requested_chunk_size, K, V); - gated_deltanet_prefill_chunked_f16(q_data, k_data, v_data, g_data, b_data, s_data, out, - B, T, Hq, Hv, K, V, chunk_size, scale); -} diff --git a/cactus/kernel/kernel_i8mm.cpp b/cactus/kernel/kernel_i8mm.cpp deleted file mode 100644 index c8c976173..000000000 --- a/cactus/kernel/kernel_i8mm.cpp +++ /dev/null @@ -1,189 +0,0 @@ -#include - -#ifndef __ARM_FEATURE_MATMUL_INT8 -#error "kernel_i8mm.cpp must be compiled with I8MM enabled (e.g. -march=armv8.2-a+fp16+simd+dotprod+i8mm)" -#endif - -#include "kernel.h" -#include "kernel_utils.h" -#include -#include - -void cactus_gemv_int8_i8mm( - const int8_t* A, - const float A_scale, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - for (size_t n_block = block_start; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - float32x4_t running_sum = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const int8_t* b_ptr = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_ptr + group_size * 8, 0, 3); - - int32x4_t acc_01 = vdupq_n_s32(0); - int32x4_t acc_23 = vdupq_n_s32(0); - - for (size_t k_off = 0; k_off < group_size; k_off += 8) { - int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); - int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); - - int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); - int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); - - int8x8_t a8 = vld1_s8(a_ptr + k_off); - int8x16_t a_dup = vcombine_s8(a8, a8); - - acc_01 = vmmlaq_s32(acc_01, a_dup, b01); - acc_23 = vmmlaq_s32(acc_23, a_dup, b23); - } - - float32x2_t r01 = vcvt_f32_s32(vget_low_s32(acc_01)); - float32x2_t r23 = vcvt_f32_s32(vget_low_s32(acc_23)); - float32x4_t group_result = vcombine_f32(r01, r23); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); - running_sum = vmlaq_f32(running_sum, group_result, scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - -void cactus_gemm_int8_i8mm( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - const size_t M_pairs = (M + 1) / 2; - const size_t total_tiles = M_pairs * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t m_pair = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m0 = m_pair * 2; - const size_t m1 = std::min(m0 + 1, M - 1); - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - const int8_t* a_row0 = A + m0 * K; - const int8_t* a_row1 = A + m1 * K; - - float32x4_t sum_row0 = vdupq_n_f32(0.0f); - float32x4_t sum_row1 = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* b_ptr = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_ptr + group_size * 8, 0, 3); - - int32x4_t acc_01 = vdupq_n_s32(0); - int32x4_t acc_23 = vdupq_n_s32(0); - - for (size_t k_off = 0; k_off < group_size; k_off += 8) { - int32x4_t chunk0 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4)); - int32x4_t chunk1 = vreinterpretq_s32_s8(vld1q_s8(b_ptr + k_off * 4 + 16)); - - int8x16_t b01 = vreinterpretq_s8_s32(vzip1q_s32(chunk0, chunk1)); - int8x16_t b23 = vreinterpretq_s8_s32(vzip2q_s32(chunk0, chunk1)); - - int8x8_t a0_8 = vld1_s8(a_row0 + k_base + k_off); - int8x8_t a1_8 = vld1_s8(a_row1 + k_base + k_off); - int8x16_t a_2rows = vcombine_s8(a0_8, a1_8); - - acc_01 = vmmlaq_s32(acc_01, a_2rows, b01); - acc_23 = vmmlaq_s32(acc_23, a_2rows, b23); - } - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float32x4_t scales = vcvt_f32_f16(vld1_f16(scale_ptr)); - - float32x2_t r0_01 = vcvt_f32_s32(vget_low_s32(acc_01)); - float32x2_t r0_23 = vcvt_f32_s32(vget_low_s32(acc_23)); - float32x4_t group_row0 = vcombine_f32(r0_01, r0_23); - sum_row0 = vmlaq_f32(sum_row0, group_row0, scales); - - float32x2_t r1_01 = vcvt_f32_s32(vget_high_s32(acc_01)); - float32x2_t r1_23 = vcvt_f32_s32(vget_high_s32(acc_23)); - float32x4_t group_row1 = vcombine_f32(r1_01, r1_23); - sum_row1 = vmlaq_f32(sum_row1, group_row1, scales); - } - - { - float32x4_t result = vmulq_n_f32(sum_row0, A_scales[m0]); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + m0 * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[m0 * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - - if (m0 + 1 < M) { - float32x4_t result = vmulq_n_f32(sum_row1, A_scales[m1]); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + m1 * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[m1 * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} diff --git a/cactus/kernel/kernel_lstm.cpp b/cactus/kernel/kernel_lstm.cpp deleted file mode 100644 index 84d0077a1..000000000 --- a/cactus/kernel/kernel_lstm.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include - -void cactus_lstm_cell_f16( - const __fp16* x_input, - const __fp16* h_prev, - const __fp16* c_prev, - const __fp16* weight_ih, - const __fp16* weight_hh, - const __fp16* bias_ih, - const __fp16* bias_hh, - __fp16* h_new, - __fp16* c_new, - size_t batch_size, - size_t input_size, - size_t hidden_size -) { - constexpr size_t SIMD_WIDTH = 8; - const size_t gate_size = 4 * hidden_size; - - std::vector<__fp16> gates_ih(batch_size * gate_size); - std::vector<__fp16> gates_hh(batch_size * gate_size); - - cactus_matmul_f16(x_input, weight_ih, gates_ih.data(), batch_size, input_size, gate_size); - cactus_matmul_f16(h_prev, weight_hh, gates_hh.data(), batch_size, hidden_size, gate_size); - - const size_t simd_end = (hidden_size / SIMD_WIDTH) * SIMD_WIDTH; - - CactusThreading::parallel_for(batch_size, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t batch_start, size_t batch_end) { - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t b = batch_start; b < batch_end; ++b) { - const size_t gate_offset = b * gate_size; - const size_t hidden_offset = b * hidden_size; - - for (size_t h = 0; h < simd_end; h += SIMD_WIDTH) { - float16x8_t i_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + h]), - vld1q_f16(&gates_hh[gate_offset + h])), - vaddq_f16(vld1q_f16(&bias_ih[h]), - vld1q_f16(&bias_hh[h]))); - - float16x8_t f_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[hidden_size + h]), - vld1q_f16(&bias_hh[hidden_size + h]))); - - float16x8_t g_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 2 * hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + 2 * hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[2 * hidden_size + h]), - vld1q_f16(&bias_hh[2 * hidden_size + h]))); - - float16x8_t o_gate = vaddq_f16(vaddq_f16(vld1q_f16(&gates_ih[gate_offset + 3 * hidden_size + h]), - vld1q_f16(&gates_hh[gate_offset + 3 * hidden_size + h])), - vaddq_f16(vld1q_f16(&bias_ih[3 * hidden_size + h]), - vld1q_f16(&bias_hh[3 * hidden_size + h]))); - - float32x4_t i_low = vcvt_f32_f16(vget_low_f16(i_gate)); - float32x4_t i_high = vcvt_f32_f16(vget_high_f16(i_gate)); - float16x8_t i_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(i_high)))))); - - float32x4_t f_low = vcvt_f32_f16(vget_low_f16(f_gate)); - float32x4_t f_high = vcvt_f32_f16(vget_high_f16(f_gate)); - float16x8_t f_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(f_high)))))); - - float32x4_t g_low = vcvt_f32_f16(vget_low_f16(g_gate)); - float32x4_t g_high = vcvt_f32_f16(vget_high_f16(g_gate)); - float16x8_t g_act = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(g_low)), - vcvt_f16_f32(fast_tanh_f32x4(g_high))); - - float32x4_t o_low = vcvt_f32_f16(vget_low_f16(o_gate)); - float32x4_t o_high = vcvt_f32_f16(vget_high_f16(o_gate)); - float16x8_t o_act = vcombine_f16(vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_low))))), - vcvt_f16_f32(vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(o_high)))))); - - float16x8_t c_prev_vec = vld1q_f16(&c_prev[hidden_offset + h]); - float16x8_t c_update = vfmaq_f16(vmulq_f16(f_act, c_prev_vec), i_act, g_act); - vst1q_f16(&c_new[hidden_offset + h], c_update); - - float32x4_t c_low = vcvt_f32_f16(vget_low_f16(c_update)); - float32x4_t c_high = vcvt_f32_f16(vget_high_f16(c_update)); - float16x8_t c_tanh = vcombine_f16(vcvt_f16_f32(fast_tanh_f32x4(c_low)), - vcvt_f16_f32(fast_tanh_f32x4(c_high))); - vst1q_f16(&h_new[hidden_offset + h], vmulq_f16(o_act, c_tanh)); - } - - for (size_t h = simd_end; h < hidden_size; ++h) { - float i_gate_val = static_cast(gates_ih[gate_offset + h] + - gates_hh[gate_offset + h] + - bias_ih[h] + bias_hh[h]); - float f_gate_val = static_cast(gates_ih[gate_offset + hidden_size + h] + - gates_hh[gate_offset + hidden_size + h] + - bias_ih[hidden_size + h] + bias_hh[hidden_size + h]); - float g_gate_val = static_cast(gates_ih[gate_offset + 2 * hidden_size + h] + - gates_hh[gate_offset + 2 * hidden_size + h] + - bias_ih[2 * hidden_size + h] + bias_hh[2 * hidden_size + h]); - float o_gate_val = static_cast(gates_ih[gate_offset + 3 * hidden_size + h] + - gates_hh[gate_offset + 3 * hidden_size + h] + - bias_ih[3 * hidden_size + h] + bias_hh[3 * hidden_size + h]); - - float i_act = 1.0f / (1.0f + expf(-i_gate_val)); - float f_act = 1.0f / (1.0f + expf(-f_gate_val)); - float g_act = tanhf(g_gate_val); - float o_act = 1.0f / (1.0f + expf(-o_gate_val)); - - float c_val = f_act * static_cast(c_prev[hidden_offset + h]) + i_act * g_act; - c_new[hidden_offset + h] = static_cast<__fp16>(c_val); - h_new[hidden_offset + h] = static_cast<__fp16>(o_act * tanhf(c_val)); - } - } - }); -} diff --git a/cactus/kernel/kernel_matmul.cpp b/cactus/kernel/kernel_matmul.cpp deleted file mode 100644 index 929663aa2..000000000 --- a/cactus/kernel/kernel_matmul.cpp +++ /dev/null @@ -1,863 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -constexpr size_t ACCELERATE_M_THRESHOLD = 4; -constexpr size_t ACCELERATE_K_THRESHOLD = 256; -#endif - -// Do NOT Remove: Uncomment for testing on various paths -// ----- -// TEMPORARY: Force fallback path for testing on DOTPROD devices -// #undef __ARM_FEATURE_DOTPROD - -#if defined(__ARM_FEATURE_DOTPROD) - #define CACTUS_DOTQ_LANE(acc, b, a, lane) vdotq_laneq_s32(acc, b, a, lane) -#else - static inline int32x4_t cactus_dotq_with_pattern(int32x4_t acc, int8x16_t b, int8x8_t a_pattern) { - int8x8_t b_lo = vget_low_s8(b); - int8x8_t b_hi = vget_high_s8(b); - - int16x8_t prod_lo = vmull_s8(b_lo, a_pattern); - int16x8_t prod_hi = vmull_s8(b_hi, a_pattern); - - int32x4_t sum_lo = vpaddlq_s16(prod_lo); - int32x4_t sum_hi = vpaddlq_s16(prod_hi); - - int32x2_t final_lo = vpadd_s32(vget_low_s32(sum_lo), vget_high_s32(sum_lo)); - int32x2_t final_hi = vpadd_s32(vget_low_s32(sum_hi), vget_high_s32(sum_hi)); - - return vaddq_s32(acc, vcombine_s32(final_lo, final_hi)); - } - - static inline int32x4_t cactus_dotq_lane0(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_lo = vget_low_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 0)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane1(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_lo = vget_low_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_lo), 1)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane2(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_hi = vget_high_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 0)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - static inline int32x4_t cactus_dotq_lane3(int32x4_t acc, int8x16_t b, int8x16_t a) { - int8x8_t a_hi = vget_high_s8(a); - int8x8_t a_pattern = vreinterpret_s8_s32(vdup_lane_s32(vreinterpret_s32_s8(a_hi), 1)); - return cactus_dotq_with_pattern(acc, b, a_pattern); - } - - #define CACTUS_DOTQ_LANE(acc, b, a, lane) cactus_dotq_lane##lane(acc, b, a) -#endif - -static inline __fp16 hsum_f16x8(float16x8_t v) { - float16x4_t lo = vget_low_f16(v); - float16x4_t hi = vget_high_f16(v); - float16x4_t sum4 = vadd_f16(lo, hi); - float16x4_t sum2 = vadd_f16(sum4, vext_f16(sum4, sum4, 2)); - float16x4_t sum1 = vadd_f16(sum2, vext_f16(sum2, sum2, 1)); - return vget_lane_f16(sum1, 0); -} - -static void cactus_matmul_f16_worker( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t /*M*/, - size_t K, - size_t N, - size_t start_row, - size_t end_row -) { - constexpr size_t TILE_M = 4; - constexpr size_t TILE_N = 4; - const size_t K16 = (K / 16) * 16; - const size_t K8 = (K / 8) * 8; - - for (size_t row_block = start_row; row_block < end_row; row_block += TILE_M) { - const size_t m_end = std::min(row_block + TILE_M, end_row); - - for (size_t col_block = 0; col_block < N; col_block += TILE_N) { - const size_t n_end = std::min(col_block + TILE_N, N); - - float16x8_t acc[TILE_M][TILE_N]; - for (size_t m = 0; m < TILE_M; ++m) - for (size_t n = 0; n < TILE_N; ++n) - acc[m][n] = vdupq_n_f16(0); - - for (size_t k = 0; k < K16; k += 16) { - float16x8_t a0_lo = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); - float16x8_t a0_hi = (row_block < m_end) ? vld1q_f16(a + row_block * K + k + 8) : vdupq_n_f16(0); - float16x8_t a1_lo = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); - float16x8_t a1_hi = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k + 8) : vdupq_n_f16(0); - float16x8_t a2_lo = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); - float16x8_t a2_hi = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k + 8) : vdupq_n_f16(0); - float16x8_t a3_lo = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); - float16x8_t a3_hi = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k + 8) : vdupq_n_f16(0); - - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - float16x8_t b_lo = vld1q_f16(b_transposed + (col_block + ni) * K + k); - float16x8_t b_hi = vld1q_f16(b_transposed + (col_block + ni) * K + k + 8); - - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_lo, b_lo); - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_hi, b_hi); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_lo, b_lo); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_hi, b_hi); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_lo, b_lo); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_hi, b_hi); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_lo, b_lo); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_hi, b_hi); - } - } - - for (size_t k = K16; k < K8; k += 8) { - float16x8_t a0_v = (row_block < m_end) ? vld1q_f16(a + row_block * K + k) : vdupq_n_f16(0); - float16x8_t a1_v = (row_block + 1 < m_end) ? vld1q_f16(a + (row_block + 1) * K + k) : vdupq_n_f16(0); - float16x8_t a2_v = (row_block + 2 < m_end) ? vld1q_f16(a + (row_block + 2) * K + k) : vdupq_n_f16(0); - float16x8_t a3_v = (row_block + 3 < m_end) ? vld1q_f16(a + (row_block + 3) * K + k) : vdupq_n_f16(0); - - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - float16x8_t b_v = vld1q_f16(b_transposed + (col_block + ni) * K + k); - acc[0][ni] = vfmaq_f16(acc[0][ni], a0_v, b_v); - acc[1][ni] = vfmaq_f16(acc[1][ni], a1_v, b_v); - acc[2][ni] = vfmaq_f16(acc[2][ni], a2_v, b_v); - acc[3][ni] = vfmaq_f16(acc[3][ni], a3_v, b_v); - } - } - - for (size_t k = K8; k < K; ++k) { - for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { - __fp16 av = a[(row_block + mi) * K + k]; - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - __fp16 bv = b_transposed[(col_block + ni) * K + k]; - acc[mi][ni] = vsetq_lane_f16(vgetq_lane_f16(acc[mi][ni], 0) + av * bv, acc[mi][ni], 0); - } - } - } - - for (size_t mi = 0; mi < TILE_M && row_block + mi < m_end; ++mi) { - for (size_t ni = 0; ni < TILE_N && col_block + ni < n_end; ++ni) { - c[(row_block + mi) * N + col_block + ni] = hsum_f16x8(acc[mi][ni]); - } - } - } - } -} - -void cactus_matmul_f16( - const __fp16* a, - const __fp16* b_transposed, - __fp16* c, - size_t M, - size_t K, - size_t N -) { - -#ifdef __APPLE__ - if (K >= ACCELERATE_K_THRESHOLD && M >= ACCELERATE_M_THRESHOLD) { - const size_t a_len = M * K; - const size_t b_len = N * K; - const size_t c_len = M * N; - - std::vector A_f32(a_len); - std::vector BT_f32(b_len); - std::vector C_f32(c_len); - - for (size_t i = 0; i < a_len; i++) A_f32[i] = (float)a[i]; - for (size_t i = 0; i < b_len; i++) BT_f32[i] = (float)b_transposed[i]; - - cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, - (int)M, (int)N, (int)K, - 1.0f, A_f32.data(), (int)K, - BT_f32.data(), (int)K, - 0.0f, C_f32.data(), (int)N); - - for (size_t i = 0; i < c_len; i++) { - float v = C_f32[i]; - if (v > 65504.f) v = 65504.f; - else if (v < -65504.f) v = -65504.f; - c[i] = (__fp16)v; - } - return; - } -#endif - - constexpr size_t TILE_M = 4; - const size_t num_row_blocks = (M + TILE_M - 1) / TILE_M; - - CactusThreading::parallel_for(num_row_blocks, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [=](size_t start_block, size_t end_block) { - for (size_t block_idx = start_block; block_idx < end_block; ++block_idx) { - size_t start_row = block_idx * TILE_M; - size_t end_row = std::min(start_row + TILE_M, M); - - cactus_matmul_f16_worker( - a, b_transposed, c, - M, K, N, - start_row, end_row - ); - - } - }); -} - - -void cactus_gemv_int8( - const int8_t* A, - const float A_scale, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - for (size_t n_block = block_start; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - - float32x4_t running_sum = vdupq_n_f32(0.0f); - - size_t g = 0; - for (; g + 1 < num_groups; g += 2) { - const size_t k_base0 = g * group_size; - const size_t k_base1 = (g + 1) * group_size; - - const int8_t* a_ptr0 = A + k_base0; - const int8_t* a_ptr1 = A + k_base1; - const int8_t* b_base0 = B + (n_block * K + k_base0) * 4; - const int8_t* b_base1 = B + (n_block * K + k_base1) * 4; - - __builtin_prefetch(b_base0 + group_size * 8, 0, 3); - - int32x4_t acc0 = vdupq_n_s32(0); - int32x4_t acc1 = vdupq_n_s32(0); - - { - int8x16_t a_vec = vld1q_s8(a_ptr0); - int8x16_t b0 = vld1q_s8(b_base0); - int8x16_t b1 = vld1q_s8(b_base0 + 16); - int8x16_t b2 = vld1q_s8(b_base0 + 32); - int8x16_t b3 = vld1q_s8(b_base0 + 48); - - acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); - acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); - acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); - acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr0 + 16); - b0 = vld1q_s8(b_base0 + 64); - b1 = vld1q_s8(b_base0 + 80); - b2 = vld1q_s8(b_base0 + 96); - b3 = vld1q_s8(b_base0 + 112); - - acc0 = CACTUS_DOTQ_LANE(acc0, b0, a_vec, 0); - acc0 = CACTUS_DOTQ_LANE(acc0, b1, a_vec, 1); - acc0 = CACTUS_DOTQ_LANE(acc0, b2, a_vec, 2); - acc0 = CACTUS_DOTQ_LANE(acc0, b3, a_vec, 3); - } - - { - int8x16_t a_vec = vld1q_s8(a_ptr1); - int8x16_t b0 = vld1q_s8(b_base1); - int8x16_t b1 = vld1q_s8(b_base1 + 16); - int8x16_t b2 = vld1q_s8(b_base1 + 32); - int8x16_t b3 = vld1q_s8(b_base1 + 48); - - acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); - acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); - acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); - acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr1 + 16); - b0 = vld1q_s8(b_base1 + 64); - b1 = vld1q_s8(b_base1 + 80); - b2 = vld1q_s8(b_base1 + 96); - b3 = vld1q_s8(b_base1 + 112); - - acc1 = CACTUS_DOTQ_LANE(acc1, b0, a_vec, 0); - acc1 = CACTUS_DOTQ_LANE(acc1, b1, a_vec, 1); - acc1 = CACTUS_DOTQ_LANE(acc1, b2, a_vec, 2); - acc1 = CACTUS_DOTQ_LANE(acc1, b3, a_vec, 3); - } - - const __fp16* scale_ptr0 = B_scales + (n_block * num_groups + g) * 4; - const __fp16* scale_ptr1 = B_scales + (n_block * num_groups + g + 1) * 4; - - float16x4_t scales0_f16 = vld1_f16(scale_ptr0); - float16x4_t scales1_f16 = vld1_f16(scale_ptr1); - float32x4_t scales0 = vcvt_f32_f16(scales0_f16); - float32x4_t scales1 = vcvt_f32_f16(scales1_f16); - - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc0), scales0); - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc1), scales1); - } - - for (; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const int8_t* b_base = B + (n_block * K + k_base) * 4; - - int32x4_t acc = vdupq_n_s32(0); - - int8x16_t a_vec = vld1q_s8(a_ptr); - int8x16_t b0 = vld1q_s8(b_base); - int8x16_t b1 = vld1q_s8(b_base + 16); - int8x16_t b2 = vld1q_s8(b_base + 32); - int8x16_t b3 = vld1q_s8(b_base + 48); - - acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); - - a_vec = vld1q_s8(a_ptr + 16); - b0 = vld1q_s8(b_base + 64); - b1 = vld1q_s8(b_base + 80); - b2 = vld1q_s8(b_base + 96); - b3 = vld1q_s8(b_base + 112); - - acc = CACTUS_DOTQ_LANE(acc, b0, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_vec, 3); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - -void cactus_gemm_int8( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - constexpr size_t TILE_M = 8; - constexpr size_t TILE_N = 4; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + TILE_N - 1) / TILE_N; - const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; - const size_t total_tiles = num_row_tiles * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t tile_row = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m_start = tile_row * TILE_M; - const size_t m_end = std::min(m_start + TILE_M, M); - const size_t n_start = n_block * TILE_N; - const size_t n_end = std::min(n_start + TILE_N, N); - const size_t actual_m = m_end - m_start; - const size_t actual_n = n_end - n_start; - - const int8_t* a_rows[TILE_M]; - for (size_t mi = 0; mi < TILE_M; mi++) { - size_t row = m_start + (mi < actual_m ? mi : actual_m - 1); - a_rows[mi] = A + row * K; - } - - float32x4_t running_sum[TILE_M]; - for (size_t mi = 0; mi < TILE_M; mi++) { - running_sum[mi] = vdupq_n_f32(0.0f); - } - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* b_base = B + (n_block * K + k_base) * 4; - - __builtin_prefetch(b_base + group_size * 4, 0, 3); - - int8x16_t b00 = vld1q_s8(b_base); - int8x16_t b01 = vld1q_s8(b_base + 16); - int8x16_t b02 = vld1q_s8(b_base + 32); - int8x16_t b03 = vld1q_s8(b_base + 48); - - int8x16_t b10 = vld1q_s8(b_base + 64); - int8x16_t b11 = vld1q_s8(b_base + 80); - int8x16_t b12 = vld1q_s8(b_base + 96); - int8x16_t b13 = vld1q_s8(b_base + 112); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - #define CACTUS_GEMM_ROW(ROW) do { \ - const int8_t* a_ptr_##ROW = a_rows[ROW] + k_base; \ - int32x4_t acc_##ROW = vdupq_n_s32(0); \ - int8x16_t a_lo_##ROW = vld1q_s8(a_ptr_##ROW); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b00, a_lo_##ROW, 0); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b01, a_lo_##ROW, 1); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b02, a_lo_##ROW, 2); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b03, a_lo_##ROW, 3); \ - int8x16_t a_hi_##ROW = vld1q_s8(a_ptr_##ROW + 16); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b10, a_hi_##ROW, 0); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b11, a_hi_##ROW, 1); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b12, a_hi_##ROW, 2); \ - acc_##ROW = CACTUS_DOTQ_LANE(acc_##ROW, b13, a_hi_##ROW, 3); \ - running_sum[ROW] = vmlaq_f32(running_sum[ROW], vcvtq_f32_s32(acc_##ROW), scales); \ - } while(0) - - CACTUS_GEMM_ROW(0); - CACTUS_GEMM_ROW(1); - CACTUS_GEMM_ROW(2); - CACTUS_GEMM_ROW(3); - CACTUS_GEMM_ROW(4); - CACTUS_GEMM_ROW(5); - CACTUS_GEMM_ROW(6); - CACTUS_GEMM_ROW(7); - #undef CACTUS_GEMM_ROW - } - - for (size_t mi = 0; mi < actual_m; mi++) { - const float a_scale = A_scales[m_start + mi]; - float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + (m_start + mi) * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} - -void cactus_matmul_int8( - const int8_t* A, - const float* A_scales, - const int8_t* B, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - -#if defined(CACTUS_COMPILE_I8MM) - if (cpu_has_i8mm()) { - if (M == 1) { - cactus_gemv_int8_i8mm(A, A_scales[0], B, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int8_i8mm(A, A_scales, B, B_scales, C, M, K, N, group_size); - } - return; - } -#endif - - if (M == 1) { - cactus_gemv_int8(A, A_scales[0], B, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); - } -} - -void cactus_gemv_int4( - const int8_t* A, - const float A_scale, - const int8_t* B_packed_raw, - const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, - size_t group_size -) { - const uint8_t* B_packed = reinterpret_cast(B_packed_raw); - if (K == 0 || N == 0) return; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + 3) / 4; - - auto process_blocks = [=](size_t block_start, size_t block_end) { - size_t n_block = block_start; - - for (; n_block + 1 < block_end; n_block += 2) { - float32x4_t sum_a = vdupq_n_f32(0.0f); - float32x4_t sum_b = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* ba = B_packed + (n_block * K + k_base) * 2; - const uint8_t* bb = B_packed + ((n_block + 1) * K + k_base) * 2; - - int32x4_t acc_a = vdupq_n_s32(0); - int32x4_t acc_b = vdupq_n_s32(0); - - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(ba, b1, b0); - unpack_int4_as_int8x16x2(ba + 16, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_lo, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_lo, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_lo, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_lo, 3); - unpack_int4_as_int8x16x2(ba + 32, b1, b0); - unpack_int4_as_int8x16x2(ba + 48, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_hi, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_hi, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_hi, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_hi, 3); - } - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(bb, b1, b0); - unpack_int4_as_int8x16x2(bb + 16, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_lo, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_lo, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_lo, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_lo, 3); - unpack_int4_as_int8x16x2(bb + 32, b1, b0); - unpack_int4_as_int8x16x2(bb + 48, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_hi, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_hi, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_hi, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_hi, 3); - } - - const __fp16* spa = B_scales + (n_block * num_groups + g) * 4; - const __fp16* spb = B_scales + ((n_block + 1) * num_groups + g) * 4; - float32x4_t sa = vcvt_f32_f16(vld1_f16(spa)); - float32x4_t sb = vcvt_f32_f16(vld1_f16(spb)); - sum_a = vmlaq_f32(sum_a, vcvtq_f32_s32(acc_a), sa); - sum_b = vmlaq_f32(sum_b, vcvtq_f32_s32(acc_b), sb); - } - - vst1_f16(C + n_block * 4, vcvt_f16_f32(vmulq_n_f32(sum_a, A_scale))); - vst1_f16(C + (n_block + 1) * 4, vcvt_f16_f32(vmulq_n_f32(sum_b, A_scale))); - } - - for (; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - const size_t actual_n = std::min(size_t(4), N - n_start); - float32x4_t running_sum = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; - - int32x4_t acc = vdupq_n_s32(0); - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(b_base, b1, b0); - unpack_int4_as_int8x16x2(b_base + 16, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_lo, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_lo, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_lo, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_lo, 3); - unpack_int4_as_int8x16x2(b_base + 32, b1, b0); - unpack_int4_as_int8x16x2(b_base + 48, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_hi, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_hi, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_hi, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_hi, 3); - - float32x4_t scales = vcvt_f32_f16(vld1_f16(B_scales + (n_block * num_groups + g) * 4)); - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); - } - - float32x4_t result = vmulq_n_f32(running_sum, A_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - if (actual_n == 4) { - vst1_f16(C + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - }; - - auto& pool = CactusThreading::get_thread_pool(); - size_t num_threads = CactusThreading::GemmThreading::get_gemv_threads(N_blocks, pool.num_workers()); - num_threads = std::min(num_threads, N_blocks); - - if (num_threads <= 1) { - process_blocks(0, N_blocks); - } else { - pool.enqueue_n_threads(N_blocks, num_threads, process_blocks); - pool.wait_all(); - } -} - -void cactus_gemv_int4_block_range( - const int8_t* A, float A_scale, - const int8_t* B_packed_raw, const __fp16* B_scales, - __fp16* C, - size_t K, size_t N, size_t group_size, - size_t block_start, size_t block_end -) { - const uint8_t* B_packed = reinterpret_cast(B_packed_raw); - if (K == 0 || N == 0 || block_end <= block_start) return; - const size_t num_groups = K / group_size; - - size_t n_block = block_start; - for (; n_block + 1 < block_end; n_block += 2) { - const size_t n_start_a = n_block * 4; - const size_t n_start_b = (n_block + 1) * 4; - float32x4_t sum_a = vdupq_n_f32(0.0f); - float32x4_t sum_b = vdupq_n_f32(0.0f); - - for (size_t g = 0; g < num_groups; ++g) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* ba = B_packed + (n_block * K + k_base) * 2; - const uint8_t* bb = B_packed + ((n_block + 1) * K + k_base) * 2; - - int32x4_t acc_a = vdupq_n_s32(0); - int32x4_t acc_b = vdupq_n_s32(0); - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(ba, b1, b0); - unpack_int4_as_int8x16x2(ba + 16, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_lo, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_lo, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_lo, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_lo, 3); - unpack_int4_as_int8x16x2(ba + 32, b1, b0); - unpack_int4_as_int8x16x2(ba + 48, b3, b2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b0, a_hi, 0); - acc_a = CACTUS_DOTQ_LANE(acc_a, b1, a_hi, 1); - acc_a = CACTUS_DOTQ_LANE(acc_a, b2, a_hi, 2); - acc_a = CACTUS_DOTQ_LANE(acc_a, b3, a_hi, 3); - } - { - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(bb, b1, b0); - unpack_int4_as_int8x16x2(bb + 16, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_lo, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_lo, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_lo, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_lo, 3); - unpack_int4_as_int8x16x2(bb + 32, b1, b0); - unpack_int4_as_int8x16x2(bb + 48, b3, b2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b0, a_hi, 0); - acc_b = CACTUS_DOTQ_LANE(acc_b, b1, a_hi, 1); - acc_b = CACTUS_DOTQ_LANE(acc_b, b2, a_hi, 2); - acc_b = CACTUS_DOTQ_LANE(acc_b, b3, a_hi, 3); - } - - const __fp16* spa = B_scales + (n_block * num_groups + g) * 4; - const __fp16* spb = B_scales + ((n_block + 1) * num_groups + g) * 4; - float32x4_t sa = vcvt_f32_f16(vld1_f16(spa)); - float32x4_t sb = vcvt_f32_f16(vld1_f16(spb)); - sum_a = vmlaq_f32(sum_a, vcvtq_f32_s32(acc_a), sa); - sum_b = vmlaq_f32(sum_b, vcvtq_f32_s32(acc_b), sb); - } - vst1_f16(C + n_start_a, vcvt_f16_f32(vmulq_n_f32(sum_a, A_scale))); - vst1_f16(C + n_start_b, vcvt_f16_f32(vmulq_n_f32(sum_b, A_scale))); - } - - for (; n_block < block_end; ++n_block) { - const size_t n_start = n_block * 4; - float32x4_t running_sum = vdupq_n_f32(0.0f); - for (size_t g = 0; g < num_groups; ++g) { - const size_t k_base = g * group_size; - const int8_t* a_ptr = A + k_base; - const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; - int32x4_t acc = vdupq_n_s32(0); - int8x16_t a_lo = vld1q_s8(a_ptr); - int8x16_t a_hi = vld1q_s8(a_ptr + 16); - int8x16_t b0, b1, b2, b3; - unpack_int4_as_int8x16x2(b_base, b1, b0); - unpack_int4_as_int8x16x2(b_base + 16, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_lo, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_lo, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_lo, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_lo, 3); - unpack_int4_as_int8x16x2(b_base + 32, b1, b0); - unpack_int4_as_int8x16x2(b_base + 48, b3, b2); - acc = CACTUS_DOTQ_LANE(acc, b0, a_hi, 0); - acc = CACTUS_DOTQ_LANE(acc, b1, a_hi, 1); - acc = CACTUS_DOTQ_LANE(acc, b2, a_hi, 2); - acc = CACTUS_DOTQ_LANE(acc, b3, a_hi, 3); - float32x4_t scales = vcvt_f32_f16(vld1_f16(B_scales + (n_block * num_groups + g) * 4)); - running_sum = vmlaq_f32(running_sum, vcvtq_f32_s32(acc), scales); - } - vst1_f16(C + n_start, vcvt_f16_f32(vmulq_n_f32(running_sum, A_scale))); - } -} - -void cactus_gemm_int4( - const int8_t* A, - const float* A_scales, - const int8_t* B_packed_raw, - const __fp16* B_scales, - __fp16* C, - size_t M, size_t K, size_t N, - size_t group_size -) { - if (M == 0 || K == 0 || N == 0) return; - - const uint8_t* B_packed = reinterpret_cast(B_packed_raw); - - constexpr size_t TILE_M = 4; - constexpr size_t TILE_N = 4; - - const size_t num_groups = K / group_size; - const size_t N_blocks = (N + TILE_N - 1) / TILE_N; - const size_t num_row_tiles = (M + TILE_M - 1) / TILE_M; - const size_t total_tiles = num_row_tiles * N_blocks; - - CactusThreading::parallel_gemm_tiles(M, total_tiles, - [=](size_t tile_start, size_t tile_end) { - for (size_t tile_idx = tile_start; tile_idx < tile_end; ++tile_idx) { - const size_t tile_row = tile_idx / N_blocks; - const size_t n_block = tile_idx % N_blocks; - const size_t m_start = tile_row * TILE_M; - const size_t m_end = std::min(m_start + TILE_M, M); - const size_t n_start = n_block * TILE_N; - const size_t n_end = std::min(n_start + TILE_N, N); - const size_t actual_m = m_end - m_start; - const size_t actual_n = n_end - n_start; - - float32x4_t running_sum[TILE_M] = { - vdupq_n_f32(0.0f), vdupq_n_f32(0.0f), - vdupq_n_f32(0.0f), vdupq_n_f32(0.0f) - }; - - for (size_t g = 0; g < num_groups; g++) { - const size_t k_base = g * group_size; - const uint8_t* b_base = B_packed + (n_block * K + k_base) * 2; - - __builtin_prefetch(b_base + group_size * 2, 0, 3); - - int8x16_t b00, b01, b02, b03; - int8x16_t b10, b11, b12, b13; - - unpack_int4_as_int8x16x2(b_base, b01, b00); - unpack_int4_as_int8x16x2(b_base + 16, b03, b02); - unpack_int4_as_int8x16x2(b_base + 32, b11, b10); - unpack_int4_as_int8x16x2(b_base + 48, b13, b12); - - const __fp16* scale_ptr = B_scales + (n_block * num_groups + g) * 4; - float16x4_t scales_f16 = vld1_f16(scale_ptr); - float32x4_t scales = vcvt_f32_f16(scales_f16); - - for (size_t mi = 0; mi < actual_m; mi++) { - const int8_t* a_ptr = A + (m_start + mi) * K + k_base; - - int32x4_t acc = vdupq_n_s32(0); - - int8x16_t a_vec = vld1q_s8(a_ptr); - acc = CACTUS_DOTQ_LANE(acc, b00, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b01, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b02, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b03, a_vec, 3); - - a_vec = vld1q_s8(a_ptr + 16); - acc = CACTUS_DOTQ_LANE(acc, b10, a_vec, 0); - acc = CACTUS_DOTQ_LANE(acc, b11, a_vec, 1); - acc = CACTUS_DOTQ_LANE(acc, b12, a_vec, 2); - acc = CACTUS_DOTQ_LANE(acc, b13, a_vec, 3); - - running_sum[mi] = vmlaq_f32(running_sum[mi], vcvtq_f32_s32(acc), scales); - } - } - - for (size_t mi = 0; mi < actual_m; mi++) { - const float a_scale = A_scales[m_start + mi]; - float32x4_t result = vmulq_n_f32(running_sum[mi], a_scale); - float16x4_t result_f16 = vcvt_f16_f32(result); - - if (actual_n == 4) { - vst1_f16(C + (m_start + mi) * N + n_start, result_f16); - } else { - for (size_t ni = 0; ni < actual_n; ni++) { - C[(m_start + mi) * N + n_start + ni] = vget_lane_f16(result_f16, 0); - result_f16 = vext_f16(result_f16, result_f16, 1); - } - } - } - } - }); -} - -void cactus_matmul_int4(const int8_t* A, const float* A_scales, - const int8_t* B_packed, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { - if (M == 0 || K == 0 || N == 0) return; - - if (M == 1) { - cactus_gemv_int4(A, A_scales[0], B_packed, B_scales, C, K, N, group_size); - } else { - cactus_gemm_int4(A, A_scales, B_packed, B_scales, C, M, K, N, group_size); - } -} - -void cactus_matmul_integer(Precision precision, - const int8_t* A, const float* A_scales, - const int8_t* B, const __fp16* B_scales, - __fp16* C, size_t M, size_t K, size_t N, size_t group_size) { - if (precision == Precision::INT4) { - cactus_matmul_int4(A, A_scales, B, B_scales, C, M, K, N, group_size); - } else { - cactus_matmul_int8(A, A_scales, B, B_scales, C, M, K, N, group_size); - } -} diff --git a/cactus/kernel/kernel_nn.cpp b/cactus/kernel/kernel_nn.cpp deleted file mode 100644 index 0bb29863d..000000000 --- a/cactus/kernel/kernel_nn.cpp +++ /dev/null @@ -1,924 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include -#include -#include -#include - -void cactus_relu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - for (size_t i = 0; i < num_elements; ++i) { - __fp16 x = input[i]; - output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(0); - } -} - -void cactus_leaky_relu_f16(const __fp16* input, __fp16* output, size_t num_elements, float negative_slope) { - for (size_t i = 0; i < num_elements; ++i) { - __fp16 x = input[i]; - output[i] = x > static_cast<__fp16>(0) ? x : static_cast<__fp16>(static_cast(x) * negative_slope); - } -} - -void cactus_silu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - const float32x4_t one_f32 = vdupq_n_f32(1.0f); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x)); - - float32x4_t exp_low = fast_exp_f32x4(vnegq_f32(x_low)); - float32x4_t exp_high = fast_exp_f32x4(vnegq_f32(x_high)); - - float32x4_t sigmoid_low = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_low)); - float32x4_t sigmoid_high = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_high)); - - float32x4_t silu_low = vmulq_f32(x_low, sigmoid_low); - float32x4_t silu_high = vmulq_f32(x_high, sigmoid_high); - - float16x8_t silu = vcombine_f16(vcvt_f16_f32(silu_low), vcvt_f16_f32(silu_high)); - vst1q_f16(&output[i], silu); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x_f32 = static_cast(input[i]); - float sigmoid = 1.0f / (1.0f + expf(-x_f32)); - output[i] = static_cast<__fp16>(x_f32 * sigmoid); - } - }); -} - -void cactus_gelu_f16(const __fp16* input, __fp16* output, size_t num_elements) { - const float sqrt_2_over_pi = 0.7978845608028654f; - const float coeff = 0.044715f; - - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - const float32x4_t half = vdupq_n_f32(0.5f); - const float32x4_t one = vdupq_n_f32(1.0f); - const float32x4_t sqrt_2_pi_vec = vdupq_n_f32(sqrt_2_over_pi); - const float32x4_t coeff_vec = vdupq_n_f32(coeff); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x_f16 = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); - - float32x4_t x_cubed_low = vmulq_f32(vmulq_f32(x_low, x_low), x_low); - float32x4_t x_cubed_high = vmulq_f32(vmulq_f32(x_high, x_high), x_high); - - float32x4_t inner_low = vfmaq_f32(x_low, coeff_vec, x_cubed_low); - float32x4_t inner_high = vfmaq_f32(x_high, coeff_vec, x_cubed_high); - inner_low = vmulq_f32(sqrt_2_pi_vec, inner_low); - inner_high = vmulq_f32(sqrt_2_pi_vec, inner_high); - - float32x4_t tanh_low = fast_tanh_f32x4(inner_low); - float32x4_t tanh_high = fast_tanh_f32x4(inner_high); - - float32x4_t gelu_low = vmulq_f32(vmulq_f32(half, x_low), vaddq_f32(one, tanh_low)); - float32x4_t gelu_high = vmulq_f32(vmulq_f32(half, x_high), vaddq_f32(one, tanh_high)); - - float16x8_t gelu_f16 = vcombine_f16(vcvt_f16_f32(gelu_low), vcvt_f16_f32(gelu_high)); - vst1q_f16(&output[i], gelu_f16); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x = static_cast(input[i]); - float inner = sqrt_2_over_pi * (x + coeff * x * x * x); - float gelu = 0.5f * x * (1.0f + tanhf(inner)); - output[i] = static_cast<__fp16>(gelu); - } - }); -} - -void cactus_gelu_f16_erf(const __fp16* input, __fp16* output, size_t num_elements) -{ - const float inv_sqrt2 = 0.70710678118654752440f; - - CactusThreading::parallel_for( - num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - - constexpr size_t SIMD = 8; - size_t vec_end = start_idx + ((end_idx - start_idx) / SIMD) * SIMD; - - for (size_t i = start_idx; i < vec_end; i += SIMD) { - - float16x8_t xh = vld1q_f16(&input[i]); - - float32x4_t x0 = vcvt_f32_f16(vget_low_f16(xh)); - float32x4_t x1 = vcvt_f32_f16(vget_high_f16(xh)); - - float32x4_t arg0 = vmulq_n_f32(x0, inv_sqrt2); - float32x4_t arg1 = vmulq_n_f32(x1, inv_sqrt2); - - float arg0_s[4], arg1_s[4]; - float erf0_s[4], erf1_s[4]; - vst1q_f32(arg0_s, arg0); - vst1q_f32(arg1_s, arg1); - - for (int j = 0; j < 4; j++) { - erf0_s[j] = erff(arg0_s[j]); - erf1_s[j] = erff(arg1_s[j]); - } - - float32x4_t erf0 = vld1q_f32(erf0_s); - float32x4_t erf1 = vld1q_f32(erf1_s); - - float32x4_t t0 = vaddq_f32(vdupq_n_f32(1.0f), erf0); - float32x4_t t1 = vaddq_f32(vdupq_n_f32(1.0f), erf1); - - float32x4_t gelu0 = vmulq_f32(vmulq_n_f32(x0, 0.5f), t0); - float32x4_t gelu1 = vmulq_f32(vmulq_n_f32(x1, 0.5f), t1); - - float16x8_t gelu_h = vcombine_f16( - vcvt_f16_f32(gelu0), - vcvt_f16_f32(gelu1) - ); - - vst1q_f16(&output[i], gelu_h); - } - - for (size_t i = vec_end; i < end_idx; i++) { - float x = (float)input[i]; - float arg = x * inv_sqrt2; - float res = 0.5f * x * (1.0f + erff(arg)); - output[i] = (__fp16)res; - } - } - ); -} - -void cactus_sigmoid_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - const float32x4_t one_f32 = vdupq_n_f32(1.0f); - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x)); - - float32x4_t exp_low = fast_exp_f32x4(vnegq_f32(x_low)); - float32x4_t exp_high = fast_exp_f32x4(vnegq_f32(x_high)); - - float32x4_t sigmoid_low = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_low)); - float32x4_t sigmoid_high = vdivq_f32(one_f32, vaddq_f32(one_f32, exp_high)); - - float16x8_t sigmoid = vcombine_f16(vcvt_f16_f32(sigmoid_low), vcvt_f16_f32(sigmoid_high)); - vst1q_f16(&output[i], sigmoid); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x_f32 = static_cast(input[i]); - float sigmoid = 1.0f / (1.0f + expf(-x_f32)); - output[i] = static_cast<__fp16>(sigmoid); - } - }); -} - -void cactus_tanh_f16(const __fp16* input, __fp16* output, size_t num_elements) { - CactusThreading::parallel_for(num_elements, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - constexpr size_t SIMD_WIDTH = 8; - const size_t vectorized_end = start_idx + ((end_idx - start_idx) / SIMD_WIDTH) * SIMD_WIDTH; - - for (size_t i = start_idx; i < vectorized_end; i += SIMD_WIDTH) { - float16x8_t x_f16 = vld1q_f16(&input[i]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_f16)); - - float32x4_t tanh_low = fast_tanh_f32x4(x_low); - float32x4_t tanh_high = fast_tanh_f32x4(x_high); - - float16x8_t tanh_f16 = vcombine_f16(vcvt_f16_f32(tanh_low), vcvt_f16_f32(tanh_high)); - vst1q_f16(&output[i], tanh_f16); - } - - for (size_t i = vectorized_end; i < end_idx; ++i) { - float x = static_cast(input[i]); - output[i] = static_cast<__fp16>(std::tanh(x)); - } - }); -} - -void cactus_glu_f16( - const __fp16* input, - __fp16* output, - size_t outer_size, - size_t split_size, - size_t inner_size -) { - if (outer_size == 0 || split_size == 0 || inner_size == 0) return; - - const size_t axis_size = split_size * 2; - const size_t rows = outer_size * split_size; - - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t row = row_start; row < row_end; ++row) { - const size_t outer_idx = row / split_size; - const size_t split_idx = row - outer_idx * split_size; - const size_t in_base = outer_idx * axis_size * inner_size; - const size_t out_base = outer_idx * split_size * inner_size; - - const __fp16* a = input + in_base + split_idx * inner_size; - const __fp16* b = input + in_base + (split_idx + split_size) * inner_size; - __fp16* y = output + out_base + split_idx * inner_size; - - size_t i = 0; - for (; i + 8 <= inner_size; i += 8) { - const float16x8_t av = vld1q_f16(a + i); - const float16x8_t bv = vld1q_f16(b + i); - - const float32x4_t a_lo = vcvt_f32_f16(vget_low_f16(av)); - const float32x4_t a_hi = vcvt_f32_f16(vget_high_f16(av)); - const float32x4_t b_lo = vcvt_f32_f16(vget_low_f16(bv)); - const float32x4_t b_hi = vcvt_f32_f16(vget_high_f16(bv)); - - const float32x4_t exp_lo = fast_exp_f32x4(vnegq_f32(b_lo)); - const float32x4_t exp_hi = fast_exp_f32x4(vnegq_f32(b_hi)); - const float32x4_t gate_lo = vdivq_f32(one, vaddq_f32(one, exp_lo)); - const float32x4_t gate_hi = vdivq_f32(one, vaddq_f32(one, exp_hi)); - - const float32x4_t y_lo = vmulq_f32(a_lo, gate_lo); - const float32x4_t y_hi = vmulq_f32(a_hi, gate_hi); - vst1q_f16(y + i, vcombine_f16(vcvt_f16_f32(y_lo), vcvt_f16_f32(y_hi))); - } - - for (; i < inner_size; ++i) { - const float gate = 1.0f / (1.0f + expf(-static_cast(b[i]))); - y[i] = static_cast<__fp16>(static_cast(a[i]) * gate); - } - } - }); -} - -void cactus_glu_f32( - const float* input, - float* output, - size_t outer_size, - size_t split_size, - size_t inner_size -) { - if (outer_size == 0 || split_size == 0 || inner_size == 0) return; - - const size_t axis_size = split_size * 2; - const size_t rows = outer_size * split_size; - - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - const float32x4_t one = vdupq_n_f32(1.0f); - - for (size_t row = row_start; row < row_end; ++row) { - const size_t outer_idx = row / split_size; - const size_t split_idx = row - outer_idx * split_size; - const size_t in_base = outer_idx * axis_size * inner_size; - const size_t out_base = outer_idx * split_size * inner_size; - - const float* a = input + in_base + split_idx * inner_size; - const float* b = input + in_base + (split_idx + split_size) * inner_size; - float* y = output + out_base + split_idx * inner_size; - - size_t i = 0; - for (; i + 4 <= inner_size; i += 4) { - const float32x4_t av = vld1q_f32(a + i); - const float32x4_t bv = vld1q_f32(b + i); - const float32x4_t expv = fast_exp_f32x4(vnegq_f32(bv)); - const float32x4_t gate = vdivq_f32(one, vaddq_f32(one, expv)); - vst1q_f32(y + i, vmulq_f32(av, gate)); - } - - for (; i < inner_size; ++i) { - const float gate = 1.0f / (1.0f + expf(-b[i])); - y[i] = a[i] * gate; - } - } - }); -} - -void cactus_batchnorm_f16( - const __fp16* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - __fp16* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -) { - if (outer_size == 0 || channels == 0 || inner_size == 0) return; - - std::vector scale(channels); - std::vector shift(channels); - for (size_t c = 0; c < channels; ++c) { - const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); - scale[c] = weight[c] * inv_std; - shift[c] = bias[c] - running_mean[c] * scale[c]; - } - - const size_t rows = outer_size * channels; - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t c = row % channels; - const float32x4_t scv = vdupq_n_f32(scale[c]); - const float32x4_t shv = vdupq_n_f32(shift[c]); - - const __fp16* x = input + row * inner_size; - __fp16* y = output + row * inner_size; - - size_t i = 0; - for (; i + 8 <= inner_size; i += 8) { - const float16x8_t xv = vld1q_f16(x + i); - float32x4_t lo = vcvt_f32_f16(vget_low_f16(xv)); - float32x4_t hi = vcvt_f32_f16(vget_high_f16(xv)); - lo = vfmaq_f32(shv, lo, scv); - hi = vfmaq_f32(shv, hi, scv); - const float16x8_t yv = vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); - vst1q_f16(y + i, yv); - } - - for (; i < inner_size; ++i) { - y[i] = static_cast<__fp16>(static_cast(x[i]) * scale[c] + shift[c]); - } - } - }); -} - -void cactus_batchnorm_f32( - const float* input, - const float* weight, - const float* bias, - const float* running_mean, - const float* running_var, - float* output, - size_t outer_size, - size_t channels, - size_t inner_size, - float epsilon -) { - if (outer_size == 0 || channels == 0 || inner_size == 0) return; - - std::vector scale(channels); - std::vector shift(channels); - for (size_t c = 0; c < channels; ++c) { - const float inv_std = 1.0f / std::sqrt(running_var[c] + epsilon); - scale[c] = weight[c] * inv_std; - shift[c] = bias[c] - running_mean[c] * scale[c]; - } - - const size_t rows = outer_size * channels; - CactusThreading::parallel_for(rows, CactusThreading::Thresholds::ELEMENT_WISE, - [&](size_t row_start, size_t row_end) { - for (size_t row = row_start; row < row_end; ++row) { - const size_t c = row % channels; - const float32x4_t scv = vdupq_n_f32(scale[c]); - const float32x4_t shv = vdupq_n_f32(shift[c]); - - const float* x = input + row * inner_size; - float* y = output + row * inner_size; - - size_t i = 0; - for (; i + 4 <= inner_size; i += 4) { - const float32x4_t xv = vld1q_f32(x + i); - vst1q_f32(y + i, vfmaq_f32(shv, xv, scv)); - } - - for (; i < inner_size; ++i) { - y[i] = x[i] * scale[c] + shift[c]; - } - } - }); -} - -void kernel_softmax_f16_single(const __fp16* input, __fp16* output, size_t vocab_size) { - - constexpr size_t SIMD_WIDTH = 8; - constexpr size_t UNROLL_FACTOR = 4; - constexpr size_t VECTORIZED_WIDTH = SIMD_WIDTH * UNROLL_FACTOR; - const size_t vocab_vectorized = (vocab_size / VECTORIZED_WIDTH) * VECTORIZED_WIDTH; - - float32x4_t max_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - max_vec[u] = vdupq_n_f32(-std::numeric_limits::infinity()); - } - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); - max_vec[u * 2] = vmaxq_f32(max_vec[u * 2], x_low); - max_vec[u * 2 + 1] = vmaxq_f32(max_vec[u * 2 + 1], x_high); - } - } - - float32x4_t final_max = max_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - final_max = vmaxq_f32(final_max, max_vec[u]); - } - - float max_val = vmaxvq_f32(final_max); - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - max_val = std::max(max_val, static_cast(input[i])); - } - - const float32x4_t max_broadcast = vdupq_n_f32(max_val); - - float32x4_t sum_vec[UNROLL_FACTOR * 2]; - for (size_t u = 0; u < UNROLL_FACTOR * 2; u++) { - sum_vec[u] = vdupq_n_f32(0.0f); - } - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t x_vec_f16 = vld1q_f16(&input[i + u * SIMD_WIDTH]); - - float32x4_t x_low = vcvt_f32_f16(vget_low_f16(x_vec_f16)); - float32x4_t x_high = vcvt_f32_f16(vget_high_f16(x_vec_f16)); - - float32x4_t exp_low = fast_exp_f32x4(vsubq_f32(x_low, max_broadcast)); - float32x4_t exp_high = fast_exp_f32x4(vsubq_f32(x_high, max_broadcast)); - - float16x8_t exp_f16 = vcombine_f16(vcvt_f16_f32(exp_low), vcvt_f16_f32(exp_high)); - vst1q_f16(&output[i + u * SIMD_WIDTH], exp_f16); - - sum_vec[u * 2] = vaddq_f32(sum_vec[u * 2], exp_low); - sum_vec[u * 2 + 1] = vaddq_f32(sum_vec[u * 2 + 1], exp_high); - } - } - - float32x4_t final_sum = sum_vec[0]; - for (size_t u = 1; u < UNROLL_FACTOR * 2; u++) { - final_sum = vaddq_f32(final_sum, sum_vec[u]); - } - - float sum = vaddvq_f32(final_sum); - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - float exp_val = expf(static_cast(input[i]) - max_val); - output[i] = static_cast<__fp16>(exp_val); - sum += exp_val; - } - - const float inv_sum = 1.0f / sum; - const float16x8_t inv_sum_vec_f16 = vdupq_n_f16(static_cast<__fp16>(inv_sum)); - - for (size_t i = 0; i < vocab_vectorized; i += VECTORIZED_WIDTH) { - for (size_t u = 0; u < UNROLL_FACTOR; u++) { - float16x8_t exp_vec = vld1q_f16(&output[i + u * SIMD_WIDTH]); - float16x8_t result = vmulq_f16(exp_vec, inv_sum_vec_f16); - vst1q_f16(&output[i + u * SIMD_WIDTH], result); - } - } - - for (size_t i = vocab_vectorized; i < vocab_size; ++i) { - output[i] = static_cast<__fp16>(static_cast(output[i]) * inv_sum); - } -} - -void cactus_softmax_f16( - const __fp16* input, - __fp16* output, - size_t batch_size, - size_t seq_len, - size_t vocab_size -) { - CactusThreading::parallel_for(batch_size * seq_len, CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start_idx, size_t end_idx) { - for (size_t idx = start_idx; idx < end_idx; ++idx) { - const size_t offset = idx * vocab_size; - kernel_softmax_f16_single(input + offset, output + offset, vocab_size); - } - }); -} - -void cactus_sample_f32(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - cactus_sample_f32_ex(logits, output, vocab_size, - temperature, top_p, 0.15f, 1.1f, - top_k, random_seed, - bias_values, bias_indices, bias_count); -} - -void cactus_sample_f32_ex(const float* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - if (vocab_size == 0) { - output[0] = 0; - return; - } - - const bool has_bias = bias_values && bias_indices && bias_count > 0; - - std::vector filtered_logits(vocab_size); - std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(float)); - - if (has_bias) { - for (size_t i = 0; i < bias_count; ++i) { - uint32_t idx = bias_indices[i]; - if (idx < vocab_size) { - filtered_logits[idx] += bias_values[i]; - } - } - } - - if (temperature == 0.0f) { - auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); - output[0] = static_cast(std::distance(filtered_logits.begin(), it)); - return; - } - - if (temperature > 0) { - for (size_t i = 0; i < vocab_size; ++i) { - filtered_logits[i] /= temperature; - } - } - - (void)repetition_penalty; - - if (top_k > 0) { - std::vector> logit_pairs; - logit_pairs.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - logit_pairs.emplace_back(filtered_logits[i], i); - } - std::sort(logit_pairs.begin(), logit_pairs.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - if (top_k < vocab_size) { - float kth_value = logit_pairs[top_k - 1].first; - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] < kth_value) { - filtered_logits[i] = -std::numeric_limits::infinity(); - } - } - } - } - - if (min_p > 0.0f) { - float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - if (!std::isinf(max_logit)) { - std::vector temp_probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (!std::isinf(filtered_logits[i])) { - temp_probs[i] = std::exp(filtered_logits[i] - max_logit); - sum += temp_probs[i]; - } else { - temp_probs[i] = 0.0f; - } - } - - if (sum > 0.0f) { - for (size_t i = 0; i < vocab_size; ++i) { - temp_probs[i] /= sum; - } - - float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); - float threshold = max_prob * min_p; - - for (size_t i = 0; i < vocab_size; ++i) { - if (temp_probs[i] < threshold) { - filtered_logits[i] = -std::numeric_limits::infinity(); - } - } - } - } - } - - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted_logits; - sorted_logits.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - if (!std::isinf(filtered_logits[i])) { - sorted_logits.emplace_back(filtered_logits[i], i); - } - } - std::sort(sorted_logits.begin(), sorted_logits.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - float max_logit = sorted_logits.empty() ? 0.0f : sorted_logits[0].first; - std::vector temp_probs; - temp_probs.reserve(sorted_logits.size()); - float sum = 0.0f; - for (const auto& pair : sorted_logits) { - float prob = std::exp(pair.first - max_logit); - temp_probs.push_back(prob); - sum += prob; - } - - for (float& prob : temp_probs) { - prob /= sum; - } - - float cumulative_prob = 0.0f; - std::vector indices_to_remove(sorted_logits.size(), false); - for (size_t i = 0; i < sorted_logits.size(); ++i) { - cumulative_prob += temp_probs[i]; - if (cumulative_prob > top_p) { - indices_to_remove[i] = true; - } - } - - if (!indices_to_remove.empty()) { - for (size_t i = 1; i < indices_to_remove.size(); ++i) { - indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; - } - indices_to_remove[0] = false; - } - - for (size_t i = 0; i < sorted_logits.size(); ++i) { - if (indices_to_remove[i]) { - filtered_logits[sorted_logits[i].second] = -std::numeric_limits::infinity(); - } - } - } - - float max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - if (std::isinf(max_logit)) { - output[0] = 0; - return; - } - - std::vector probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (std::isinf(filtered_logits[i])) { - probs[i] = 0.0f; - } else { - probs[i] = std::exp(filtered_logits[i] - max_logit); - sum += probs[i]; - } - } - - if (sum == 0.0f) { - output[0] = 0; - return; - } - - for (size_t i = 0; i < vocab_size; ++i) { - probs[i] /= sum; - } - - uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; - std::mt19937 gen(actual_seed); - std::uniform_real_distribution dist(0.0f, 1.0f); - float sample = dist(gen); - - float cumulative = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - cumulative += probs[i]; - if (cumulative >= sample) { - output[0] = static_cast(i); - return; - } - } - - for (size_t i = vocab_size; i > 0; --i) { - if (probs[i-1] > 0.0f) { - output[0] = static_cast(i-1); - return; - } - } - - output[0] = 0; -} - -void cactus_sample_f16(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - cactus_sample_f16_ex(logits, output, vocab_size, - temperature, top_p, 0.15f, 1.1f, - top_k, random_seed, - bias_values, bias_indices, bias_count); -} - -void cactus_sample_f16_ex(const __fp16* logits, uint32_t* output, size_t vocab_size, - float temperature, float top_p, float min_p, float repetition_penalty, - size_t top_k, size_t random_seed, - const float* bias_values, const uint32_t* bias_indices, - size_t bias_count) { - if (vocab_size == 0) { - output[0] = 0; - return; - } - - const bool has_bias = bias_values && bias_indices && bias_count > 0; - - std::vector<__fp16> filtered_logits(vocab_size); - std::memcpy(filtered_logits.data(), logits, vocab_size * sizeof(__fp16)); - - if (has_bias) { - for (size_t i = 0; i < bias_count; ++i) { - uint32_t idx = bias_indices[i]; - if (idx < vocab_size) { - filtered_logits[idx] = static_cast<__fp16>(static_cast(filtered_logits[idx]) + bias_values[i]); - } - } - } - - if (temperature == 0.0f) { - auto it = std::max_element(filtered_logits.begin(), filtered_logits.end()); - output[0] = static_cast(std::distance(filtered_logits.begin(), it)); - return; - } - - if (temperature > 0) { - __fp16 inv_temp = static_cast<__fp16>(1.0f / temperature); - float16x8_t inv_temp_vec = vdupq_n_f16(inv_temp); - size_t i = 0; - for (; i + 8 <= vocab_size; i += 8) { - float16x8_t logits_vec = vld1q_f16(&filtered_logits[i]); - float16x8_t scaled = vmulq_f16(logits_vec, inv_temp_vec); - vst1q_f16(&filtered_logits[i], scaled); - } - for (; i < vocab_size; ++i) { - filtered_logits[i] = filtered_logits[i] * inv_temp; - } - } - - (void)repetition_penalty; - - if (top_k > 0) { - std::vector> logit_pairs; - logit_pairs.reserve(vocab_size); - for (size_t i = 0; i < vocab_size; ++i) { - logit_pairs.emplace_back(filtered_logits[i], i); - } - std::partial_sort(logit_pairs.begin(), - logit_pairs.begin() + std::min(top_k, vocab_size), - logit_pairs.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - if (top_k < vocab_size) { - __fp16 kth_value = logit_pairs[top_k - 1].first; - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] < kth_value) { - filtered_logits[i] = neg_inf; - } - } - } - } - - if (min_p > 0.0f) { - float max_logit = -std::numeric_limits::infinity(); - for (size_t i = 0; i < vocab_size; ++i) { - max_logit = std::max(max_logit, static_cast(filtered_logits[i])); - } - if (!std::isinf(max_logit)) { - std::vector temp_probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - float v = static_cast(filtered_logits[i]); - if (!std::isinf(v)) { - temp_probs[i] = std::exp(v - max_logit); - sum += temp_probs[i]; - } else { - temp_probs[i] = 0.0f; - } - } - if (sum > 0.0f) { - for (float& p : temp_probs) p /= sum; - float max_prob = *std::max_element(temp_probs.begin(), temp_probs.end()); - float threshold = max_prob * min_p; - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (temp_probs[i] < threshold) { - filtered_logits[i] = neg_inf; - } - } - } - } - } - - if (top_p > 0.0f && top_p < 1.0f) { - std::vector> sorted_logits; - sorted_logits.reserve(vocab_size); - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] != neg_inf) { - sorted_logits.emplace_back(filtered_logits[i], i); - } - } - std::sort(sorted_logits.begin(), sorted_logits.end(), - [](const auto& a, const auto& b) { return a.first > b.first; }); - - __fp16 max_logit = sorted_logits.empty() ? static_cast<__fp16>(0.0f) : sorted_logits[0].first; - std::vector temp_probs; - temp_probs.reserve(sorted_logits.size()); - float sum = 0.0f; - for (const auto& pair : sorted_logits) { - float prob = std::exp(static_cast(pair.first - max_logit)); - temp_probs.push_back(prob); - sum += prob; - } - - for (float& prob : temp_probs) { - prob /= sum; - } - - float cumulative_prob = 0.0f; - std::vector indices_to_remove(sorted_logits.size(), false); - bool threshold_reached = false; - for (size_t i = 0; i < sorted_logits.size(); ++i) { - cumulative_prob += temp_probs[i]; - if (cumulative_prob > top_p && i > 0) { - threshold_reached = true; - } - if (threshold_reached) { - indices_to_remove[i] = true; - } - } - - if (!indices_to_remove.empty()) { - for (size_t i = 1; i < indices_to_remove.size(); ++i) { - indices_to_remove[i] = indices_to_remove[i-1] || indices_to_remove[i]; - } - indices_to_remove[0] = false; - } - - for (size_t i = 0; i < sorted_logits.size(); ++i) { - if (indices_to_remove[i]) { - filtered_logits[sorted_logits[i].second] = neg_inf; - } - } - } - - __fp16 max_logit = *std::max_element(filtered_logits.begin(), filtered_logits.end()); - __fp16 neg_inf = static_cast<__fp16>(-std::numeric_limits::infinity()); - if (max_logit == neg_inf) { - output[0] = 0; - return; - } - - std::vector probs(vocab_size); - float sum = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - if (filtered_logits[i] == neg_inf) { - probs[i] = 0.0f; - } else { - probs[i] = std::exp(static_cast(filtered_logits[i] - max_logit)); - sum += probs[i]; - } - } - - if (sum == 0.0f) { - output[0] = 0; - return; - } - - for (size_t i = 0; i < vocab_size; ++i) { - probs[i] /= sum; - } - - uint32_t actual_seed = (random_seed == 0) ? std::random_device{}() : random_seed; - std::mt19937 gen(actual_seed); - std::uniform_real_distribution dist(0.0f, 1.0f); - float sample = dist(gen); - - float cumulative = 0.0f; - for (size_t i = 0; i < vocab_size; ++i) { - cumulative += probs[i]; - if (cumulative >= sample) { - output[0] = static_cast(i); - return; - } - } - - for (size_t i = vocab_size; i > 0; --i) { - if (probs[i-1] > 0.0f) { - output[0] = static_cast(i-1); - return; - } - } - - output[0] = 0; -} diff --git a/cactus/kernel/kernel_quants.cpp b/cactus/kernel/kernel_quants.cpp deleted file mode 100644 index d3553603b..000000000 --- a/cactus/kernel/kernel_quants.cpp +++ /dev/null @@ -1,312 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include -#include -#include - -void cactus_int8_to_fp32(const int8_t* src, float* dst, size_t count, float scale) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 16) * 16; - float32x4_t scale_vec = vdupq_n_f32(scale); - - for (size_t i = start; i < simd_end; i += 16) { - int8x16_t input = vld1q_s8(&src[i]); - - int16x8_t low = vmovl_s8(vget_low_s8(input)); - int16x8_t high = vmovl_s8(vget_high_s8(input)); - - int32x4_t low_low = vmovl_s16(vget_low_s16(low)); - int32x4_t low_high = vmovl_s16(vget_high_s16(low)); - int32x4_t high_low = vmovl_s16(vget_low_s16(high)); - int32x4_t high_high = vmovl_s16(vget_high_s16(high)); - - float32x4_t f_low_low = vcvtq_f32_s32(low_low); - float32x4_t f_low_high = vcvtq_f32_s32(low_high); - float32x4_t f_high_low = vcvtq_f32_s32(high_low); - float32x4_t f_high_high = vcvtq_f32_s32(high_high); - - vst1q_f32(&dst[i], vmulq_f32(f_low_low, scale_vec)); - vst1q_f32(&dst[i + 4], vmulq_f32(f_low_high, scale_vec)); - vst1q_f32(&dst[i + 8], vmulq_f32(f_high_low, scale_vec)); - vst1q_f32(&dst[i + 12], vmulq_f32(f_high_high, scale_vec)); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast(src[i]) * scale; - } - }); -} - -void cactus_fp32_to_int8(const float* src, int8_t* dst, size_t count, float scale) { - const float inv_scale = 1.0f / scale; - - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, inv_scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 16) * 16; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t min_vec = vdupq_n_f32(-128.0f); - float32x4_t max_vec = vdupq_n_f32(127.0f); - - for (size_t i = start; i < simd_end; i += 16) { - float32x4_t input_0 = vld1q_f32(&src[i]); - float32x4_t input_1 = vld1q_f32(&src[i + 4]); - float32x4_t input_2 = vld1q_f32(&src[i + 8]); - float32x4_t input_3 = vld1q_f32(&src[i + 12]); - - float32x4_t scaled_0 = vmulq_f32(input_0, inv_scale_vec); - float32x4_t scaled_1 = vmulq_f32(input_1, inv_scale_vec); - float32x4_t scaled_2 = vmulq_f32(input_2, inv_scale_vec); - float32x4_t scaled_3 = vmulq_f32(input_3, inv_scale_vec); - - scaled_0 = vmaxq_f32(vminq_f32(scaled_0, max_vec), min_vec); - scaled_1 = vmaxq_f32(vminq_f32(scaled_1, max_vec), min_vec); - scaled_2 = vmaxq_f32(vminq_f32(scaled_2, max_vec), min_vec); - scaled_3 = vmaxq_f32(vminq_f32(scaled_3, max_vec), min_vec); - - int32x4_t int_0 = vcvtnq_s32_f32(scaled_0); - int32x4_t int_1 = vcvtnq_s32_f32(scaled_1); - int32x4_t int_2 = vcvtnq_s32_f32(scaled_2); - int32x4_t int_3 = vcvtnq_s32_f32(scaled_3); - - int16x8_t int16_low = vcombine_s16(vqmovn_s32(int_0), vqmovn_s32(int_1)); - int16x8_t int16_high = vcombine_s16(vqmovn_s32(int_2), vqmovn_s32(int_3)); - - int8x16_t result = vcombine_s8(vqmovn_s16(int16_low), vqmovn_s16(int16_high)); - vst1q_s8(&dst[i], result); - } - - for (size_t i = simd_end; i < end; ++i) { - float quantized = src[i] * inv_scale; - dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); - } - }); -} - -void cactus_fp16_to_fp32(const __fp16* src, float* dst, size_t count) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - - for (size_t i = start; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t output_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t output_high = vcvt_f32_f16(vget_high_f16(input)); - - vst1q_f32(&dst[i], output_low); - vst1q_f32(&dst[i + 4], output_high); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast(src[i]); - } - }); -} - -void cactus_fp32_to_fp16(const float* src, __fp16* dst, size_t count) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - - for (size_t i = start; i < simd_end; i += 8) { - float32x4_t input_low = vld1q_f32(&src[i]); - float32x4_t input_high = vld1q_f32(&src[i + 4]); - - float16x4_t output_low = vcvt_f16_f32(input_low); - float16x4_t output_high = vcvt_f16_f32(input_high); - - float16x8_t output = vcombine_f16(output_low, output_high); - vst1q_f16(&dst[i], output); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast<__fp16>(src[i]); - } - }); -} - -void cactus_int8_to_fp16(const int8_t* src, __fp16* dst, size_t count, float scale) { - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - float32x4_t scale_vec = vdupq_n_f32(scale); - - for (size_t i = start; i < simd_end; i += 8) { - int8x8_t input = vld1_s8(&src[i]); - - int16x8_t int16 = vmovl_s8(input); - int32x4_t int32_low = vmovl_s16(vget_low_s16(int16)); - int32x4_t int32_high = vmovl_s16(vget_high_s16(int16)); - - float32x4_t float_low = vcvtq_f32_s32(int32_low); - float32x4_t float_high = vcvtq_f32_s32(int32_high); - - float_low = vmulq_f32(float_low, scale_vec); - float_high = vmulq_f32(float_high, scale_vec); - - float16x8_t output = vcombine_f16(vcvt_f16_f32(float_low), vcvt_f16_f32(float_high)); - vst1q_f16(&dst[i], output); - } - - for (size_t i = simd_end; i < end; ++i) { - dst[i] = static_cast<__fp16>(static_cast(src[i]) * scale); - } - }); -} - -void cactus_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count, float scale) { - const float inv_scale = 1.0f / scale; - - CactusThreading::parallel_for(count, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, inv_scale](size_t start, size_t end) { - const size_t simd_end = start + ((end - start) / 8) * 8; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - float32x4_t min_vec = vdupq_n_f32(-128.0f); - float32x4_t max_vec = vdupq_n_f32(127.0f); - - for (size_t i = start; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); - - float32x4_t scaled_low = vmulq_f32(input_low, inv_scale_vec); - float32x4_t scaled_high = vmulq_f32(input_high, inv_scale_vec); - - scaled_low = vmaxq_f32(vminq_f32(scaled_low, max_vec), min_vec); - scaled_high = vmaxq_f32(vminq_f32(scaled_high, max_vec), min_vec); - - int32x4_t int_low = vcvtnq_s32_f32(scaled_low); - int32x4_t int_high = vcvtnq_s32_f32(scaled_high); - - int16x8_t int16_combined = vcombine_s16(vqmovn_s32(int_low), vqmovn_s32(int_high)); - int8x8_t result = vqmovn_s16(int16_combined); - - vst1_s8(&dst[i], result); - } - - for (size_t i = simd_end; i < end; ++i) { - float quantized = static_cast(src[i]) * inv_scale; - dst[i] = static_cast(std::round(std::max(-128.0f, std::min(127.0f, quantized)))); - } - }); -} - -float cactus_fp16_max_abs(const __fp16* src, size_t count) { - float32x4_t abs_max_vec = vdupq_n_f32(0.0f); - const size_t simd_end = (count / 8) * 8; - - for (size_t i = 0; i < simd_end; i += 8) { - float16x8_t input = vld1q_f16(&src[i]); - - float32x4_t input_low = vcvt_f32_f16(vget_low_f16(input)); - float32x4_t input_high = vcvt_f32_f16(vget_high_f16(input)); - - float32x4_t abs_low = vabsq_f32(input_low); - float32x4_t abs_high = vabsq_f32(input_high); - - abs_max_vec = vmaxq_f32(abs_max_vec, abs_low); - abs_max_vec = vmaxq_f32(abs_max_vec, abs_high); - } - - float max_abs = vmaxvq_f32(abs_max_vec); - - for (size_t i = simd_end; i < count; ++i) { - float abs_val = std::abs(static_cast(src[i])); - max_abs = std::max(max_abs, abs_val); - } - - return max_abs; -} - -static inline float quantize_group_fp16_to_int8(const __fp16* src, int8_t* dst, size_t count) { - - float32x4_t max_vec = vdupq_n_f32(0.0f); - size_t k = 0; - - for (; k + 8 <= count; k += 8) { - float16x8_t vals = vld1q_f16(src + k); - max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_low_f16(vals)))); - max_vec = vmaxq_f32(max_vec, vabsq_f32(vcvt_f32_f16(vget_high_f16(vals)))); - } - - float max_abs = vmaxvq_f32(max_vec); - for (; k < count; k++) { - float val = std::abs(static_cast(src[k])); - if (val > max_abs) max_abs = val; - } - - float scale = max_abs / 127.0f; - if (scale < 1e-10f) scale = 1e-10f; - float inv_scale = 1.0f / scale; - float32x4_t inv_scale_vec = vdupq_n_f32(inv_scale); - - k = 0; - for (; k + 8 <= count; k += 8) { - float16x8_t vals = vld1q_f16(src + k); - int32x4_t i0 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_low_f16(vals)), inv_scale_vec)); - int32x4_t i1 = vcvtnq_s32_f32(vmulq_f32(vcvt_f32_f16(vget_high_f16(vals)), inv_scale_vec)); - int16x4_t s0 = vqmovn_s32(i0); - int16x4_t s1 = vqmovn_s32(i1); - int8x8_t result = vqmovn_s16(vcombine_s16(s0, s1)); - vst1_s8(dst + k, result); - } - - for (; k < count; k++) { - float val = static_cast(src[k]) * inv_scale; - int32_t q = static_cast(roundf(val)); - q = std::max(-128, std::min(127, q)); - dst[k] = static_cast(q); - } - - return scale; -} - -void cactus_quantize_kv_fp16_to_int8( - const __fp16* src, - int8_t* dst, - float* scales, - size_t seq_len, size_t kv_heads, size_t head_dim, - size_t group_size -) { - const size_t num_groups = (head_dim + group_size - 1) / group_size; - - CactusThreading::parallel_for(seq_len * kv_heads, CactusThreading::Thresholds::ELEMENT_WISE, - [src, dst, scales, head_dim, group_size, num_groups](size_t start, size_t end) { - for (size_t idx = start; idx < end; idx++) { - const __fp16* head_src = src + idx * head_dim; - int8_t* head_dst = dst + idx * head_dim; - float* head_scales = scales + idx * num_groups; - - for (size_t g = 0; g < num_groups; g++) { - size_t group_start = g * group_size; - size_t group_count = std::min(group_size, head_dim - group_start); - head_scales[g] = quantize_group_fp16_to_int8( - head_src + group_start, - head_dst + group_start, - group_count - ); - } - } - }); -} - -void cactus_unpack_int4_to_int8(const uint8_t* packed, int8_t* unpacked, size_t unpacked_count) { - assert (unpacked_count % 32 == 0 && "Unpacked count must be a multiple of 32 for int4 unpacking"); - const size_t num_groups = unpacked_count / 32; - - CactusThreading::parallel_for(num_groups, CactusThreading::Thresholds::ELEMENT_WISE, - [packed, unpacked](size_t start, size_t end) { - for (size_t group = start; group < end; ++group) { - const uint8_t* src = &packed[group * 16]; - int8_t* dst = &unpacked[group * 32]; - - int8x16_t high_signed, low_signed; - unpack_int4_as_int8x16x2(src, high_signed, low_signed); - vst1q_s8(dst, low_signed); - vst1q_s8(dst + 16, high_signed); - } - }); -} diff --git a/cactus/kernel/kernel_reduce.cpp b/cactus/kernel/kernel_reduce.cpp deleted file mode 100644 index e0f3d19e8..000000000 --- a/cactus/kernel/kernel_reduce.cpp +++ /dev/null @@ -1,327 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include - -template -static void axis_reduce_f32_impl(const __fp16* input, __fp16* output, - size_t outer_size, size_t axis_size, size_t inner_size, - FinalizeFn finalize) { - if (inner_size == 1) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t start_outer, size_t end_outer) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - - for (size_t outer = start_outer; outer < end_outer; ++outer) { - const __fp16* row = input + outer * axis_size; - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - - for (size_t a = 0; a < vec_axis; a += W) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(row + a), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - } - - float total = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); - for (size_t a = vec_axis; a < axis_size; ++a) { - total += static_cast(row[a]); - } - - output[outer] = static_cast<__fp16>(finalize(total, axis_size)); - } - }); - return; - } - - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - - for (size_t a = 0; a < vec_axis; a += W) { - __fp16 values[W]; - for (size_t j = 0; j < W; j++) { - values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; - } - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(values), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - } - - float total = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); - - for (size_t a = vec_axis; a < axis_size; a++) { - total += static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); - } - - output[outer * inner_size + inner] = static_cast<__fp16>(finalize(total, axis_size)); - }); -} - -template -static void axis_reduce_f16_impl(const __fp16* input, __fp16* output, - size_t outer_size, size_t axis_size, size_t inner_size, - float16x8_t init_vec, __fp16 init_scalar, - SimdReduceOp simd_reduce, ScalarReduceOp scalar_reduce) { - if (inner_size == 1) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t start_outer, size_t end_outer) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - - for (size_t outer = start_outer; outer < end_outer; ++outer) { - const __fp16* row = input + outer * axis_size; - float16x8_t acc = init_vec; - - for (size_t a = 0; a < vec_axis; a += W) { - acc = simd_reduce(acc, vld1q_f16(row + a)); - } - - __fp16 result = init_scalar; - __fp16 arr[W]; - vst1q_f16(arr, acc); - for (size_t j = 0; j < W; ++j) { - result = scalar_reduce(result, arr[j]); - } - - for (size_t a = vec_axis; a < axis_size; ++a) { - result = scalar_reduce(result, row[a]); - } - - output[outer] = result; - } - }); - return; - } - - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - float16x8_t acc = init_vec; - - for (size_t a = 0; a < vec_axis; a += W) { - __fp16 values[W]; - for (size_t j = 0; j < W; j++) { - values[j] = input[outer * axis_size * inner_size + (a + j) * inner_size + inner]; - } - acc = simd_reduce(acc, vld1q_f16(values)); - } - - __fp16 result = init_scalar; - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) { - result = scalar_reduce(result, arr[j]); - } - - for (size_t a = vec_axis; a < axis_size; a++) { - result = scalar_reduce(result, input[outer * axis_size * inner_size + a * inner_size + inner]); - } - - output[outer * inner_size + inner] = result; - }); -} - -double cactus_sum_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> double { - const size_t vec_end = start + simd_align(end - start); - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - } - - double s = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); - for (size_t i = vec_end; i < end; ++i) s += static_cast(data[i]); - return s; - }, - 0.0, [](double a, double b) { return a + b; } - ); -} - -void cactus_sum_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t) { return total; }); -} - -double cactus_mean_all_f16(const __fp16* data, size_t num_elements) { - return cactus_sum_all_f16(data, num_elements) / static_cast(num_elements); -} - -void cactus_mean_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f32_impl(input, output, outer_size, axis_size, inner_size, - [](float total, size_t axis_size) { return total / static_cast(axis_size); }); -} - -struct VarianceState { - double sum; - double sum_sq; - VarianceState() : sum(0.0), sum_sq(0.0) {} - VarianceState(double s, double sq) : sum(s), sum_sq(sq) {} -}; - -double cactus_variance_all_f16(const __fp16* data, size_t num_elements) { - VarianceState result = CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> VarianceState { - const size_t vec_end = start + simd_align(end - start); - - float32x4_t sum_lo = vdupq_n_f32(0.0f), sum_hi = vdupq_n_f32(0.0f); - float32x4_t sq_lo = vdupq_n_f32(0.0f), sq_hi = vdupq_n_f32(0.0f); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(&data[i]), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - sq_lo = vfmaq_f32(sq_lo, lo, lo); - sq_hi = vfmaq_f32(sq_hi, hi, hi); - } - - double sum = static_cast(vaddvq_f32(vaddq_f32(sum_lo, sum_hi))); - double sum_sq = static_cast(vaddvq_f32(vaddq_f32(sq_lo, sq_hi))); - - for (size_t i = vec_end; i < end; ++i) { - double x = static_cast(data[i]); - sum += x; - sum_sq += x * x; - } - return VarianceState(sum, sum_sq); - }, - VarianceState(), - [](const VarianceState& a, const VarianceState& b) { - return VarianceState(a.sum + b.sum, a.sum_sq + b.sum_sq); - } - ); - - double mean = result.sum / static_cast(num_elements); - double mean_sq = result.sum_sq / static_cast(num_elements); - return mean_sq - mean * mean; -} - -void cactus_variance_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - if (inner_size == 1) { - CactusThreading::parallel_for(outer_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t start_outer, size_t end_outer) { - constexpr size_t W = SIMD_F16_WIDTH; - const size_t vec_axis = simd_align(axis_size); - - for (size_t outer = start_outer; outer < end_outer; ++outer) { - const __fp16* row = input + outer * axis_size; - float32x4_t sum_lo = vdupq_n_f32(0.0f); - float32x4_t sum_hi = vdupq_n_f32(0.0f); - float32x4_t sq_lo = vdupq_n_f32(0.0f); - float32x4_t sq_hi = vdupq_n_f32(0.0f); - - for (size_t a = 0; a < vec_axis; a += W) { - float32x4_t lo, hi; - f16x8_split_f32(vld1q_f16(row + a), lo, hi); - sum_lo = vaddq_f32(sum_lo, lo); - sum_hi = vaddq_f32(sum_hi, hi); - sq_lo = vfmaq_f32(sq_lo, lo, lo); - sq_hi = vfmaq_f32(sq_hi, hi, hi); - } - - float sum = vaddvq_f32(vaddq_f32(sum_lo, sum_hi)); - float sum_sq = vaddvq_f32(vaddq_f32(sq_lo, sq_hi)); - for (size_t a = vec_axis; a < axis_size; ++a) { - float x = static_cast(row[a]); - sum += x; - sum_sq += x * x; - } - - float mean = sum / static_cast(axis_size); - float mean_sq = sum_sq / static_cast(axis_size); - output[outer] = static_cast<__fp16>(mean_sq - mean * mean); - } - }); - return; - } - - CactusThreading::parallel_for_2d(outer_size, inner_size, CactusThreading::Thresholds::AXIS_REDUCE, - [&](size_t outer, size_t inner) { - float sum = 0.0f, sum_sq = 0.0f; - for (size_t a = 0; a < axis_size; a++) { - float x = static_cast(input[outer * axis_size * inner_size + a * inner_size + inner]); - sum += x; - sum_sq += x * x; - } - float mean = sum / static_cast(axis_size); - float mean_sq = sum_sq / static_cast(axis_size); - output[outer * inner_size + inner] = static_cast<__fp16>(mean_sq - mean * mean); - }); -} - -__fp16 cactus_min_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> __fp16 { - const size_t vec_end = start + simd_align(end - start); - float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(65504.0f)); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - acc = vminq_f16(acc, vld1q_f16(&data[i])); - } - - __fp16 result = static_cast<__fp16>(65504.0f); - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) result = std::min(result, arr[j]); - for (size_t i = vec_end; i < end; ++i) result = std::min(result, data[i]); - return result; - }, - static_cast<__fp16>(65504.0f), - [](__fp16 a, __fp16 b) { return std::min(a, b); } - ); -} - -void cactus_min_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, - vdupq_n_f16(static_cast<__fp16>(65504.0f)), static_cast<__fp16>(65504.0f), - [](float16x8_t a, float16x8_t b) { return vminq_f16(a, b); }, - [](__fp16 a, __fp16 b) { return std::min(a, b); }); -} - -__fp16 cactus_max_all_f16(const __fp16* data, size_t num_elements) { - return CactusThreading::parallel_reduce( - num_elements, CactusThreading::Thresholds::ALL_REDUCE, - [&](size_t start, size_t end) -> __fp16 { - const size_t vec_end = start + simd_align(end - start); - float16x8_t acc = vdupq_n_f16(static_cast<__fp16>(-65504.0f)); - - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - acc = vmaxq_f16(acc, vld1q_f16(&data[i])); - } - - __fp16 result = static_cast<__fp16>(-65504.0f); - __fp16 arr[8]; - vst1q_f16(arr, acc); - for (int j = 0; j < 8; j++) result = std::max(result, arr[j]); - for (size_t i = vec_end; i < end; ++i) result = std::max(result, data[i]); - return result; - }, - static_cast<__fp16>(-65504.0f), - [](__fp16 a, __fp16 b) { return std::max(a, b); } - ); -} - -void cactus_max_axis_f16(const __fp16* input, __fp16* output, size_t outer_size, size_t axis_size, size_t inner_size) { - axis_reduce_f16_impl(input, output, outer_size, axis_size, inner_size, - vdupq_n_f16(static_cast<__fp16>(-65504.0f)), static_cast<__fp16>(-65504.0f), - [](float16x8_t a, float16x8_t b) { return vmaxq_f16(a, b); }, - [](__fp16 a, __fp16 b) { return std::max(a, b); }); -} diff --git a/cactus/kernel/kernel_scalar.cpp b/cactus/kernel/kernel_scalar.cpp deleted file mode 100644 index a35fea26e..000000000 --- a/cactus/kernel/kernel_scalar.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "kernel.h" -#include "kernel_utils.h" -#include -#include - -void cactus_scalar_op_f16(const __fp16* input, __fp16* output, size_t num_elements, float scalar_value, ScalarOpType op_type) { - const __fp16 scalar_f16 = static_cast<__fp16>(scalar_value); - const bool use_streaming = num_elements >= STREAMING_STORE_THRESHOLD; - - switch (op_type) { - case ScalarOpType::ADD: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vaddq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v + scalar_f16); }); - break; - } - case ScalarOpType::SUBTRACT: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vsubq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v - scalar_f16); }); - break; - } - case ScalarOpType::MULTIPLY: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vmulq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v * scalar_f16); }); - break; - } - case ScalarOpType::DIVIDE: { - const float16x8_t sv = vdupq_n_f16(scalar_f16); - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [sv](float16x8_t v) { return vdivq_f16(v, sv); }, - [scalar_f16](__fp16 v) { return static_cast<__fp16>(v / scalar_f16); }); - break; - } - case ScalarOpType::ABS: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_BASIC, - [](float16x8_t v) { return vabsq_f16(v); }, - [](__fp16 v) { return static_cast<__fp16>(std::abs(static_cast(v))); }); - break; - - case ScalarOpType::EXP: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [](float16x8_t v) { return apply_f32_op_on_f16x8(v, fast_exp_f32x4); }, - [](__fp16 v) { return static_cast<__fp16>(std::exp(static_cast(v))); }, 2); - break; - - case ScalarOpType::SQRT: - elementwise_op_f16(input, output, num_elements, use_streaming, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [](float16x8_t v) { return apply_f32_op_on_f16x8(v, vsqrtq_f32); }, - [](__fp16 v) { return static_cast<__fp16>(std::sqrt(static_cast(v))); }, 2); - break; - - case ScalarOpType::POW: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>( - std::pow(static_cast(input[i]), static_cast(scalar_f16))); - } - }); - break; - - case ScalarOpType::COS: - case ScalarOpType::SIN: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - float val = static_cast(input[i]); - float result = (op_type == ScalarOpType::COS) ? std::cos(val) : std::sin(val); - output[i] = static_cast<__fp16>(result); - } - }); - break; - - case ScalarOpType::LOG: - CactusThreading::parallel_for(num_elements, - CactusThreading::Thresholds::SCALAR_EXPENSIVE, - [&](size_t start, size_t end) { - for (size_t i = start; i < end; ++i) { - output[i] = static_cast<__fp16>(std::log(static_cast(input[i]))); - } - }); - break; - } -} diff --git a/cactus/kernel/kernel_utils.h b/cactus/kernel/kernel_utils.h deleted file mode 100644 index b20215818..000000000 --- a/cactus/kernel/kernel_utils.h +++ /dev/null @@ -1,681 +0,0 @@ -#ifndef KERNEL_UTILS_H -#define KERNEL_UTILS_H - -#include -#if defined(__APPLE__) -#include -#include -#endif -#if defined(__ANDROID__) -#include -#include -#include -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -constexpr size_t NEON_VECTOR_SIZE = 16; -constexpr size_t STREAMING_STORE_THRESHOLD = 32768; - -inline void stream_store_f16x8(__fp16* dst, float16x8_t val) { -#if defined(__aarch64__) - float16x4_t lo = vget_low_f16(val); - float16x4_t hi = vget_high_f16(val); - __asm__ __volatile__( - "stnp %d0, %d1, [%2]" - : - : "w"(lo), "w"(hi), "r"(dst) - : "memory" - ); -#else - vst1q_f16(dst, val); -#endif -} - -inline bool cpu_has_i8mm() { -#if defined(__aarch64__) - static std::once_flag once; - static bool has = false; - - std::call_once(once, []() { -#if defined(__APPLE__) - int ret = 0; - size_t size = sizeof(ret); - if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &ret, &size, nullptr, 0) == 0) { - has = (ret == 1); - } -#elif defined(__ANDROID__) - unsigned long hwcap2 = getauxval(AT_HWCAP2); - #ifndef HWCAP2_I8MM - #define HWCAP2_I8MM (1 << 13) - #endif - has = (hwcap2 & HWCAP2_I8MM) != 0; -#endif - }); - - return has; -#else - return false; -#endif -} - -inline float32x4_t fast_exp_f32x4(float32x4_t x) { - const float32x4_t log2e = vdupq_n_f32(1.4426950408889634f); - const float32x4_t ln2 = vdupq_n_f32(0.6931471805599453f); - - const float32x4_t c0 = vdupq_n_f32(1.0f); - const float32x4_t c1 = vdupq_n_f32(0.6931471805599453f); - const float32x4_t c2 = vdupq_n_f32(0.2402265069591007f); - const float32x4_t c3 = vdupq_n_f32(0.05550410866482158f); - const float32x4_t c4 = vdupq_n_f32(0.009618129842071803f); - - x = vmaxq_f32(x, vdupq_n_f32(-87.0f)); - x = vminq_f32(x, vdupq_n_f32(87.0f)); - - float32x4_t z = vmulq_f32(x, log2e); - - int32x4_t zi = vcvtq_s32_f32(z); - float32x4_t zf = vsubq_f32(z, vcvtq_f32_s32(zi)); - - uint32x4_t neg_mask = vcltq_f32(zf, vdupq_n_f32(0.0f)); - zi = vsubq_s32(zi, vandq_s32(vreinterpretq_s32_u32(neg_mask), vdupq_n_s32(1))); - zf = vaddq_f32(zf, vreinterpretq_f32_u32(vandq_u32(neg_mask, vreinterpretq_u32_f32(vdupq_n_f32(1.0f))))); - - float32x4_t zf_ln2 = vmulq_f32(zf, ln2); - float32x4_t p = c4; - p = vfmaq_f32(c3, p, zf_ln2); - p = vfmaq_f32(c2, p, zf_ln2); - p = vfmaq_f32(c1, p, zf_ln2); - p = vfmaq_f32(c0, p, zf_ln2); - - int32x4_t exp_bits = vshlq_n_s32(vaddq_s32(zi, vdupq_n_s32(127)), 23); - float32x4_t scale = vreinterpretq_f32_s32(exp_bits); - - return vmulq_f32(p, scale); -} - -// Cephes-style 13/6 rational tanh approximation (same coefficients as Eigen). -// Constants are stored as static splatted arrays so the compiler emits a single -// pc-relative `ldr q` per load. -alignas(16) inline constexpr float kFastTanhAlpha[7][4] = { - { 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f, 4.89352455891786e-03f }, - { 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f, 6.37261928875436e-04f }, - { 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f, 1.48572235717979e-05f }, - { 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f, 5.12229709037114e-08f }, - {-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f,-8.60467152213735e-11f }, - { 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f, 2.00018790482477e-13f }, - {-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f,-2.76076847742355e-16f }, -}; -alignas(16) inline constexpr float kFastTanhBeta[4][4] = { - { 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f, 4.89352518554385e-03f }, - { 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f, 2.26843463243900e-03f }, - { 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f, 1.18534705686654e-04f }, - { 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f, 1.19825839466702e-06f }, -}; -alignas(16) inline constexpr float kFastTanhClampHi[4] = { 9.0f, 9.0f, 9.0f, 9.0f }; -alignas(16) inline constexpr float kFastTanhClampLo[4] = {-9.0f,-9.0f,-9.0f,-9.0f }; - -inline float32x4_t fast_tanh_f32x4(float32x4_t x) { - x = vmaxq_f32(vld1q_f32(kFastTanhClampLo), vminq_f32(vld1q_f32(kFastTanhClampHi), x)); - float32x4_t x2 = vmulq_f32(x, x); - float32x4_t p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[5]), vld1q_f32(kFastTanhAlpha[6]), x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[4]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[3]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[2]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[1]), p, x2); - p = vfmaq_f32(vld1q_f32(kFastTanhAlpha[0]), p, x2); - p = vmulq_f32(p, x); - float32x4_t q = vfmaq_f32(vld1q_f32(kFastTanhBeta[2]), vld1q_f32(kFastTanhBeta[3]), x2); - q = vfmaq_f32(vld1q_f32(kFastTanhBeta[1]), q, x2); - q = vfmaq_f32(vld1q_f32(kFastTanhBeta[0]), q, x2); - return vdivq_f32(p, q); -} - -constexpr size_t SIMD_F16_WIDTH = 8; - -inline size_t simd_align(size_t count, size_t width = SIMD_F16_WIDTH) { - return (count / width) * width; -} - -inline void f16x8_split_f32(float16x8_t v, float32x4_t& lo, float32x4_t& hi) { - lo = vcvt_f32_f16(vget_low_f16(v)); - hi = vcvt_f32_f16(vget_high_f16(v)); -} - -inline float16x8_t f32_merge_f16(float32x4_t lo, float32x4_t hi) { - return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); -} - -inline float32x4_t fast_sigmoid_f32x4(float32x4_t x) { - const float32x4_t one = vdupq_n_f32(1.0f); - return vdivq_f32(one, vaddq_f32(one, fast_exp_f32x4(vnegq_f32(x)))); -} - -template -inline float16x8_t apply_f32_op_on_f16x8(float16x8_t v, F32x4Op op) { - float32x4_t lo, hi; - f16x8_split_f32(v, lo, hi); - return f32_merge_f16(op(lo), op(hi)); -} - -inline void unpack_int4_as_int8x16x2(const uint8_t* ptr, int8x16_t& high_decoded, int8x16_t& low_decoded) { - int8x16_t packed = vreinterpretq_s8_u8(vld1q_u8(ptr)); - high_decoded = vshrq_n_s8(packed, 4); - low_decoded = vshrq_n_s8(vshlq_n_s8(packed, 4), 4); -} - -namespace CactusThreading { - -#if defined(__ANDROID__) - struct CoreTopology { - std::vector performance_cores; - std::vector all_cores; - - static CoreTopology& get() { - static CoreTopology topo = detect(); - return topo; - } - - private: - static int read_sysfs_int(const char* path) { - std::ifstream f(path); - if (!f.is_open()) return -1; - int val = -1; - f >> val; - return val; - } - - static CoreTopology detect() { - CoreTopology topo; - constexpr int MAX_CPUS = 16; - std::vector> core_caps; - - for (int i = 0; i < MAX_CPUS; ++i) { - char path[128]; - - snprintf(path, sizeof(path), - "/sys/devices/system/cpu/cpu%d/cpu_capacity", i); - int cap = read_sysfs_int(path); - if (cap > 0) { - core_caps.push_back({i, cap}); - topo.all_cores.push_back(i); - continue; - } - - snprintf(path, sizeof(path), - "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_max_freq", i); - int freq = read_sysfs_int(path); - if (freq > 0) { - core_caps.push_back({i, freq}); - topo.all_cores.push_back(i); - } - } - - if (core_caps.empty()) return topo; - - int max_cap = 0; - for (auto& [id, cap] : core_caps) { - max_cap = std::max(max_cap, cap); - } - - int threshold = static_cast(max_cap * 0.70); - for (auto& [id, cap] : core_caps) { - if (cap >= threshold) { - topo.performance_cores.push_back(id); - } - } - - return topo; - } - }; - - inline bool pin_current_thread_to_cores(const std::vector& cores) { - if (cores.empty()) return false; - cpu_set_t mask; - CPU_ZERO(&mask); - for (int core : cores) { - CPU_SET(core, &mask); - } - return sched_setaffinity(0, sizeof(mask), &mask) == 0; - } -#endif - - class ThreadPool { - private: - static constexpr size_t MAX_WORKERS = 16; - - std::vector workers; - std::deque> tasks; - - std::mutex mutex; - std::condition_variable work_available; - std::condition_variable work_done; - - bool stop{false}; - std::atomic pending_tasks{0}; - size_t num_workers_; - - void worker_thread() { - while (true) { - std::function task; - { - std::unique_lock lock(mutex); - work_available.wait(lock, [this] { - return stop || !tasks.empty(); - }); - - if (stop && tasks.empty()) { - return; - } - - task = std::move(tasks.front()); - tasks.pop_front(); - } - - task(); - - if (pending_tasks.fetch_sub(1, std::memory_order_acq_rel) == 1) { - std::lock_guard lock(mutex); - work_done.notify_one(); - } - } - } - - public: - explicit ThreadPool(size_t num_threads = std::thread::hardware_concurrency()) - : stop(false), pending_tasks(0) { - num_workers_ = std::min(num_threads, MAX_WORKERS); - if (num_workers_ == 0) num_workers_ = 1; - -#if defined(__ANDROID__) - auto& topo = CoreTopology::get(); - if (!topo.performance_cores.empty()) { - num_workers_ = std::min(num_workers_, topo.performance_cores.size()); - } -#endif - - workers.reserve(num_workers_); - for (size_t i = 0; i < num_workers_; ++i) { - workers.emplace_back([this]() { -#if defined(__ANDROID__) - auto& perf = CoreTopology::get().performance_cores; - if (!perf.empty()) { - pin_current_thread_to_cores(perf); - } -#endif - worker_thread(); - }); - } - } - - ~ThreadPool() { - { - std::lock_guard lock(mutex); - stop = true; - } - work_available.notify_all(); - for (auto& worker : workers) { - if (worker.joinable()) { - worker.join(); - } - } - } - - template - auto enqueue(F&& f) -> std::future { - using return_type = decltype(f()); - - auto task = std::make_shared>( - std::forward(f) - ); - - std::future res = task->get_future(); - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(1, std::memory_order_relaxed); - tasks.emplace_back([task](){ (*task)(); }); - } - work_available.notify_one(); - - return res; - } - - template - void enqueue_batch(size_t total_work, F task_func) { - if (total_work == 0) return; - - const size_t num_tasks = std::min(num_workers_, total_work); - const size_t per_worker = total_work / num_tasks; - const size_t remainder = total_work % num_tasks; - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(num_tasks, std::memory_order_relaxed); - - for (size_t w = 0; w < num_tasks; ++w) { - size_t start = w * per_worker + std::min(w, remainder); - size_t end = start + per_worker + (w < remainder ? 1 : 0); - tasks.emplace_back([=]() { task_func(start, end); }); - } - } - work_available.notify_all(); - } - - void wait_all() { - std::unique_lock lock(mutex); - work_done.wait(lock, [this] { - return pending_tasks.load(std::memory_order_acquire) == 0; - }); - } - - template - void enqueue_n_threads(size_t total_work, size_t num_threads, F task_func) { - if (total_work == 0 || num_threads == 0) return; - - num_threads = std::min(num_threads, std::min(num_workers_, total_work)); - const size_t per_thread = total_work / num_threads; - const size_t remainder = total_work % num_threads; - - { - std::lock_guard lock(mutex); - pending_tasks.fetch_add(num_threads, std::memory_order_relaxed); - - for (size_t t = 0; t < num_threads; ++t) { - size_t start = t * per_thread + std::min(t, remainder); - size_t end = start + per_thread + (t < remainder ? 1 : 0); - tasks.emplace_back([=]() { task_func(start, end); }); - } - } - work_available.notify_all(); - } - - size_t num_workers() const { return num_workers_; } - }; - - inline ThreadPool& get_thread_pool() { - static ThreadPool pool; - return pool; - } - - struct ParallelConfig { - size_t min_work_gate; - size_t work_per_thread; - - constexpr ParallelConfig(size_t gate, size_t per_thread) - : min_work_gate(gate), work_per_thread(per_thread) {} - }; - - inline size_t get_optimal_thread_count(size_t total_work, ParallelConfig config) { - if (total_work < config.min_work_gate) return 1; - - size_t pool_size = get_thread_pool().num_workers(); - size_t num_threads = (total_work + config.work_per_thread - 1) / config.work_per_thread; - return std::min(pool_size, std::max(static_cast(1), num_threads)); - } - - struct Thresholds { - #if defined(__ANDROID__) - static constexpr ParallelConfig ATTENTION{64, 32}; - static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; - static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; - static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; - static constexpr ParallelConfig SCALAR_BASIC{30000, 15000}; - static constexpr ParallelConfig SCALAR_EXPENSIVE{10000, 5000}; - #else // Apple - static constexpr ParallelConfig ATTENTION{32, 16}; - static constexpr ParallelConfig ELEMENT_WISE{5000, 2500}; - static constexpr ParallelConfig AXIS_REDUCE{1000, 500}; - static constexpr ParallelConfig ALL_REDUCE{10000, 5000}; - static constexpr ParallelConfig SCALAR_BASIC{5000, 2500}; - static constexpr ParallelConfig SCALAR_EXPENSIVE{2500, 1250}; - #endif - }; - - struct GemmThreading { - #if defined(__ANDROID__) - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return 1; - return pool_size; - } - static size_t get_gemv_threads(size_t /*N_blocks*/, size_t /*pool_size*/) { - return 1; - } - #elif defined(__APPLE__) && TARGET_OS_IPHONE - static constexpr size_t GEMV_MIN_N_BLOCKS = 512; - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return std::min(pool_size, static_cast(2)); - return pool_size; - } - static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { - if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; - return std::min(pool_size, static_cast(3)); - } - #else - static constexpr size_t GEMV_MIN_N_BLOCKS = 256; - static size_t get_num_threads(size_t M, size_t pool_size) { - if (M <= 1) return std::min(pool_size, static_cast(4)); - return pool_size; - } - static size_t get_gemv_threads(size_t N_blocks, size_t pool_size) { - if (N_blocks < GEMV_MIN_N_BLOCKS) return 1; - if (N_blocks < 512) return std::min(pool_size, static_cast(2)); - return std::min(pool_size, static_cast(5)); - } - #endif - }; - - inline size_t& get_gemm_thread_override() { - static size_t override_threads = 0; - return override_threads; - } - - inline void set_gemm_threads(size_t num_threads) { - get_gemm_thread_override() = num_threads; - } - - inline void reset_gemm_threads() { - get_gemm_thread_override() = 0; - } - - class TaskHandle { - private: - std::vector> futures_; - bool auto_wait_; - - public: - TaskHandle(bool auto_wait = true) : auto_wait_(auto_wait) {} - - ~TaskHandle() { - if (auto_wait_) { - wait(); - } - } - - TaskHandle(TaskHandle&&) = default; - TaskHandle& operator=(TaskHandle&&) = default; - TaskHandle(const TaskHandle&) = delete; - TaskHandle& operator=(const TaskHandle&) = delete; - - void add_future(std::future&& f) { - futures_.push_back(std::move(f)); - } - - void wait() { - for (auto& f : futures_) { - if (f.valid()) { - f.wait(); - } - } - futures_.clear(); - } - - bool is_ready() const { - for (const auto& f : futures_) { - if (f.valid() && f.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { - return false; - } - } - return true; - } - - size_t task_count() const { return futures_.size(); } - }; - - template - TaskHandle parallel_for(size_t total_work, ParallelConfig config, WorkFunc work_func, bool wait = true) { - const size_t num_threads = get_optimal_thread_count(total_work, config); - TaskHandle handle(!wait); - - if (num_threads == 1) { - if (wait) { - work_func(0, total_work); - return handle; - } - auto& pool = get_thread_pool(); - handle.add_future(pool.enqueue([work_func, total_work]() { - work_func(0, total_work); - })); - return handle; - } - - auto& pool = get_thread_pool(); - const size_t work_per_thread = total_work / num_threads; - - for (size_t t = 0; t < num_threads; ++t) { - handle.add_future(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() { - const size_t start_idx = t * work_per_thread; - const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; - work_func(start_idx, end_idx); - })); - } - - if (wait) { - handle.wait(); - } - return handle; - } - - template - void parallel_for_2d(size_t outer_size, size_t inner_size, ParallelConfig config, WorkFunc work_func) { - const size_t total_work = outer_size * inner_size; - parallel_for(total_work, config, [&](size_t start_idx, size_t end_idx) { - for (size_t work_idx = start_idx; work_idx < end_idx; ++work_idx) { - const size_t outer = work_idx / inner_size; - const size_t inner = work_idx % inner_size; - work_func(outer, inner); - } - }); - } - - template - ResultType parallel_reduce(size_t total_work, ParallelConfig config, - WorkFunc work_func, ResultType init_value, CombineFunc combine_func) { - const size_t num_threads = get_optimal_thread_count(total_work, config); - - if (num_threads == 1) { - return work_func(0, total_work); - } - - auto& pool = get_thread_pool(); - std::vector> futures; - std::vector partial_results(num_threads, init_value); - const size_t work_per_thread = total_work / num_threads; - - for (size_t t = 0; t < num_threads; ++t) { - futures.push_back(pool.enqueue([work_func, t, num_threads, work_per_thread, total_work]() -> ResultType { - const size_t start_idx = t * work_per_thread; - const size_t end_idx = (t == num_threads - 1) ? total_work : (t + 1) * work_per_thread; - return work_func(start_idx, end_idx); - })); - } - - ResultType result = init_value; - for (auto& future : futures) { - result = combine_func(result, future.get()); - } - return result; - } - - template - void parallel_gemm_tiles(size_t M, size_t total_tiles, WorkFunc work_func) { - auto& pool = get_thread_pool(); - - size_t override = get_gemm_thread_override(); - size_t num_threads = (override > 0) ? override : GemmThreading::get_num_threads(M, pool.num_workers()); - num_threads = std::min(num_threads, total_tiles); - - if (num_threads <= 1) { - work_func(0, total_tiles); - return; - } - - pool.enqueue_n_threads(total_tiles, num_threads, work_func); - pool.wait_all(); - } - -} - -template -void elementwise_op_f16(const __fp16* input, __fp16* output, size_t num_elements, - bool use_streaming, CactusThreading::ParallelConfig config, - SimdOp simd_op, ScalarOp scalar_op, size_t unroll = 4) { - CactusThreading::parallel_for(num_elements, config, - [&](size_t start, size_t end) { - const size_t n = end - start; - const size_t vec_end = start + simd_align(n); - - if (use_streaming && unroll >= 4) { - const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 4); - for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 4) { - __builtin_prefetch(&input[i + 256], 0, 0); - float16x8_t v0 = simd_op(vld1q_f16(&input[i])); - float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); - float16x8_t v2 = simd_op(vld1q_f16(&input[i + 16])); - float16x8_t v3 = simd_op(vld1q_f16(&input[i + 24])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - stream_store_f16x8(&output[i + 16], v2); - stream_store_f16x8(&output[i + 24], v3); - } - for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { - stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } else if (use_streaming && unroll >= 2) { - const size_t unrolled_end = start + simd_align(n, SIMD_F16_WIDTH * 2); - for (size_t i = start; i < unrolled_end; i += SIMD_F16_WIDTH * 2) { - __builtin_prefetch(&input[i + 128], 0, 0); - float16x8_t v0 = simd_op(vld1q_f16(&input[i])); - float16x8_t v1 = simd_op(vld1q_f16(&input[i + 8])); - stream_store_f16x8(&output[i], v0); - stream_store_f16x8(&output[i + 8], v1); - } - for (size_t i = unrolled_end; i < vec_end; i += SIMD_F16_WIDTH) { - stream_store_f16x8(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } else { - for (size_t i = start; i < vec_end; i += SIMD_F16_WIDTH) { - vst1q_f16(&output[i], simd_op(vld1q_f16(&input[i]))); - } - } - for (size_t i = vec_end; i < end; ++i) { - output[i] = scalar_op(input[i]); - } - }); -} - -#endif // KERNEL_UTILS_H diff --git a/cactus/models/gemma4/model_gemma4.cpp b/cactus/models/gemma4/model_gemma4.cpp deleted file mode 100644 index c99917878..000000000 --- a/cactus/models/gemma4/model_gemma4.cpp +++ /dev/null @@ -1,624 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4Model::Gemma4Model() : Model() {} - -Gemma4Model::Gemma4Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -bool Gemma4Model::is_global_layer(uint32_t idx) const { - return config_.layer_types[idx] == "global"; -} - -std::vector Gemma4Model::get_kv_layer_dims() const { - uint32_t n = config_.num_layers; - - std::vector dims(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - dims[i] = 0; - } else if (is_global_layer(i)) { - dims[i] = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - } else { - dims[i] = config_.attention_head_dim; - } - } - return dims; -} - -std::vector Gemma4Model::get_kv_layer_heads() const { - uint32_t n = config_.num_layers; - - std::vector heads(n); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) { - heads[i] = 0; - } else if (is_global_layer(i) && config_.num_global_kv_heads > 0) { - heads[i] = config_.num_global_kv_heads; - } else { - heads[i] = config_.attention_kv_heads; - } - } - return heads; -} - -void Gemma4Model::compact_kv_cache() { - uint32_t n = config_.num_layers; - - std::vector target_windows(n, 0); - for (uint32_t i = 0; i < n; i++) { - if (i >= first_shared_layer_) continue; - if (!is_global_layer(i)) - target_windows[i] = config_.sliding_window; - } - kv_cache_.compact_to_windows(target_windows); -} - -void Gemma4Model::post_init() { - uint32_t n = config_.num_layers; - - kv_cache_.set_window_size(0, 0); - - kv_share_map_.resize(n, -1); - shared_k_nodes_.resize(n, 0); - shared_v_nodes_.resize(n, 0); - - for (uint32_t i = first_shared_layer_; i < n; i++) { - bool is_global = is_global_layer(i); - for (int j = static_cast(first_shared_layer_) - 1; j >= 0; j--) { - if (is_global_layer(j) == is_global) { - kv_share_map_[i] = j; - break; - } - } - } -} - -void Gemma4Model::load_weights_to_graph(CactusGraph* gb) { - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - first_shared_layer_ = (n > num_shared) ? n - num_shared : n; - - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - bool has_pli = config_.hidden_size_per_layer_input > 0; - if (has_pli) { - weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); - weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); - weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); - } - - bool has_moe = config_.enable_moe_block; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_shared = (i >= first_shared_layer_); - - layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); - layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); - bool k_eq_v = config_.attention_k_eq_v && is_global_layer(i); - layer.attn_v_weight = is_shared ? 0 : (k_eq_v ? layer.attn_k_weight : gb->mmap_weights(prefix + "attn_v.weights")); - layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); - if (has_pli) { - layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); - layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); - layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); - } - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - - if (has_moe) { - uint32_t num_experts = config_.num_experts; - uint32_t hidden = config_.hidden_dim; - uint32_t expert_dim = config_.expert_intermediate_size > 0 ? config_.expert_intermediate_size : config_.ffn_intermediate_dim; - layer.moe_w1_experts.resize(num_experts); - layer.moe_w3_experts.resize(num_experts); - layer.moe_w2_experts.resize(num_experts); - - auto w1_packed = gb->mmap_weights(prefix + "moe_gate_proj.weights"); - auto w3_packed = gb->mmap_weights(prefix + "moe_up_proj.weights"); - auto w2_packed = gb->mmap_weights(prefix + "moe_down_proj.weights"); - - const auto& w1_buf = gb->get_output_buffer(w1_packed); - const auto& w3_buf = gb->get_output_buffer(w3_packed); - const auto& w2_buf = gb->get_output_buffer(w2_packed); - - auto setup_experts = [&](const BufferDesc& buf, - std::vector& expert_nodes, - size_t out_dim, size_t in_dim) { - auto* base = static_cast(const_cast(buf.get_data())); - Precision prec = buf.precision; - bool is_quantized = PrecisionTraits::is_integer(prec) && buf.group_size > 0; - size_t K = in_dim; - if (is_quantized && K % buf.group_size != 0) - K = ((K + buf.group_size - 1) / buf.group_size) * buf.group_size; - size_t expert_data_bytes = PrecisionTraits::packed_size_of(prec, out_dim * K); - size_t groups_per_expert = is_quantized ? (out_dim * K + buf.group_size - 1) / buf.group_size : 0; - size_t expert_scales_bytes = groups_per_expert * sizeof(__fp16); - - char* scales_base = is_quantized ? static_cast(const_cast(buf.scales_data)) : nullptr; - - for (uint32_t e = 0; e < num_experts; e++) { - expert_nodes[e] = gb->input({out_dim, K}, prec); - gb->set_external_input(expert_nodes[e], base + e * expert_data_bytes, prec); - if (is_quantized) { - gb->set_grouped_scales(expert_nodes[e], buf.group_size, groups_per_expert, - scales_base + e * expert_scales_bytes); - if (buf.is_interleaved) - gb->set_interleaved(expert_nodes[e], true, out_dim); - } - } - }; - - setup_experts(w1_buf, layer.moe_w1_experts, expert_dim, hidden); - setup_experts(w3_buf, layer.moe_w3_experts, expert_dim, hidden); - setup_experts(w2_buf, layer.moe_w2_experts, hidden, expert_dim); - - layer.router_proj = gb->mmap_weights(prefix + "router_proj.weights"); - layer.router_scale = gb->mmap_weights(prefix + "router_scale.weights"); - layer.moe_per_expert_scale = gb->mmap_weights(prefix + "moe_per_expert_scale.weights"); - layer.post_ffn_norm_1 = gb->mmap_weights(prefix + "post_ffn_norm_1.weights"); - layer.pre_ffn_norm_2 = gb->mmap_weights(prefix + "pre_ffn_norm_2.weights"); - layer.post_ffn_norm_2 = gb->mmap_weights(prefix + "post_ffn_norm_2.weights"); - } - } - - size_t sliding_head_dim = config_.attention_head_dim; - size_t global_head_dim = config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2; - v_norm_ones_weight_.assign(std::max(sliding_head_dim, global_head_dim), static_cast<__fp16>(1.0f)); - v_norm_ones_node_ = gb->input({sliding_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); - v_norm_ones_global_node_ = gb->input({global_head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_global_node_, v_norm_ones_weight_.data(), Precision::FP16); - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - if (std::filesystem::exists(npu_prefill_path)) { - if (!load_npu_prefill(npu_prefill_path) || !has_npu_prefill()) { - CACTUS_LOG_WARN("npu", "[gemma4] found model.mlpackage but failed to enable NPU prefill; using CPU prefill"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4] model.mlpackage not found; using CPU prefill"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4] NPU backend unavailable on this device; using CPU prefill"); - } -} - -size_t Gemma4Model::build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto gate = gb->gelu(gb->matmul(hidden, layer.per_layer_gate, true, backend)); - auto pli_slice = gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim); - auto gated = gb->multiply(gate, pli_slice); - auto pli_proj = gb->matmul(gated, layer.per_layer_proj, true, backend); - auto pli_normed = gb->rms_norm(pli_proj, layer.post_per_layer_norm, config_.layer_norm_eps); - - return gb->add(hidden, pli_normed); -} - -size_t Gemma4Model::apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset) { - if (rot_dim < head_dim) { - size_t half_dim = head_dim / 2; - size_t half_rot = rot_dim / 2; - size_t pass_len = half_dim - half_rot; - float adjusted_theta = std::pow(rope_freq, static_cast(rot_dim) / static_cast(head_dim)); - - auto left_rot = gb->slice(tensor, 3, 0, half_rot); - auto left_pass = gb->slice(tensor, 3, half_rot, pass_len); - auto right_rot = gb->slice(tensor, 3, half_dim, half_rot); - auto right_pass = gb->slice(tensor, 3, half_dim + half_rot, pass_len); - - auto rotated = gb->rope(gb->concat(left_rot, right_rot, 3), adjusted_theta, position_offset); - auto rotated_left = gb->slice(rotated, 3, 0, half_rot); - auto rotated_right = gb->slice(rotated, 3, half_rot, half_rot); - - auto new_left = gb->concat(rotated_left, left_pass, 3); - auto new_right = gb->concat(rotated_right, right_pass, 3); - return gb->concat(new_left, new_right, 3); - } - return gb->rope(tensor, rope_freq, position_offset); -} - -size_t Gemma4Model::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t seq_len = gb->get_output_buffer(input).shape[0]; - int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; - - bool is_global = is_global_layer(layer_idx); - - size_t head_dim = is_global ? (config_.global_head_dim > 0 ? config_.global_head_dim : config_.attention_head_dim * 2) : config_.attention_head_dim; - size_t num_heads = config_.attention_heads; - size_t kv_heads = is_global && config_.num_global_kv_heads > 0 ? config_.num_global_kv_heads : config_.attention_kv_heads; - float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; - size_t window = is_global ? 0 : config_.sliding_window; - size_t rot_dim = static_cast(head_dim * (is_global ? config_.global_partial_rotary_factor : 1.0f)); - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - size_t q4 = apply_partial_rope(gb, q, head_dim, rot_dim, rope_freq, position_offset); - - size_t k4, v4; - if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { - k4 = shared_k_nodes_[share_src]; - v4 = shared_v_nodes_[share_src]; - } else { - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - k4 = apply_partial_rope(gb, k, head_dim, rot_dim, rope_freq, position_offset); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - size_t v_ones = is_global ? v_norm_ones_global_node_ : v_norm_ones_node_; - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), v_ones, config_.layer_norm_eps); - v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - shared_k_nodes_[layer_idx] = k4; - shared_v_nodes_[layer_idx] = v4; - } - - if (use_cache && share_src < 0) { - cache_k_output_nodes_[layer_idx] = k4; - cache_v_output_nodes_[layer_idx] = v4; - } - - size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; - size_t attn; - if (use_cache && !kv_cache_.is_empty()) { - attn = gb->attention_int8_hybrid(q4, k4, v4, attention_scale_, position_offset, - kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), - kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), - kv_cache_.current_seq_len, kv_heads, head_dim, window); - } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); - } - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); -} - -size_t Gemma4Model::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - static const bool dense_mlp_fused_disabled = []() { - const char* env = std::getenv("CACTUS_DISABLE_DENSE_MLP_FUSED"); - return env && env[0] && env[0] != '0'; - }(); - if (!dense_mlp_fused_disabled) { - const auto& gate_buf = gb->get_output_buffer(layer.ffn_gate_weight); - const auto& up_buf = gb->get_output_buffer(layer.ffn_up_weight); - const auto& down_buf = gb->get_output_buffer(layer.ffn_down_weight); - if (gate_buf.precision == Precision::INT4 && gate_buf.group_size > 0 && - up_buf.precision == Precision::INT4 && up_buf.group_size > 0 && - down_buf.precision == Precision::INT4 && down_buf.group_size > 0) { - return gb->dense_mlp_int4_fused(input, - layer.ffn_gate_weight, - layer.ffn_up_weight, - layer.ffn_down_weight); - } - } - auto gate = gb->gelu(gb->matmul(input, layer.ffn_gate_weight, true, backend)); - auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.ffn_down_weight, true, backend); -} - -size_t Gemma4Model::build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto router_logits = gb->matmul(input, layer.router_proj, true, backend); - auto topk_result = gb->topk(router_logits, config_.num_experts_per_tok); - auto topk_indices = gb->index(topk_result, 0, 0); - auto routing_probs = gb->softmax(router_logits); - - return gb->moe_layer(input, routing_probs, topk_indices, - layer.moe_w1_experts, layer.moe_w3_experts, layer.moe_w2_experts, - config_.num_experts, config_.num_experts_per_tok, - true, 1e-6f, 1.0f, Activation::GELU, - layer.moe_per_expert_scale); -} - -size_t Gemma4Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - if (config_.enable_moe_block) { - auto h1 = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - h1 = build_mlp(gb, h1, layer_idx, backend); - h1 = gb->rms_norm(h1, layer.post_ffn_norm_1, config_.layer_norm_eps); - - auto h2 = gb->multiply(residual, layer.router_scale); - h2 = gb->rms_norm(h2, layer.pre_ffn_norm_2, config_.layer_norm_eps); - h2 = build_moe(gb, h2, layer_idx, backend); - h2 = gb->rms_norm(h2, layer.post_ffn_norm_2, config_.layer_norm_eps); - - auto combined = gb->rms_norm(gb->add(h1, h2), layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - return gb->add(residual, combined); - } - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - return gb->add(residual, mlp); -} - -size_t Gemma4Model::apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, pos_offset); - if (config_.hidden_size_per_layer_input > 0) - hidden = build_per_layer_input(gb, hidden, pli, layer_idx, backend); - if (weight_nodes_.layers[layer_idx].layer_scalar != 0) - hidden = gb->multiply(hidden, weight_nodes_.layers[layer_idx].layer_scalar); - return hidden; -} - -size_t Gemma4Model::build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend) { - uint32_t num_layers = config_.num_layers; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_proj = gb->scalar_multiply(gb->matmul(hidden, weight_nodes_.per_layer_model_proj, true, backend), - 1.0f / std::sqrt(static_cast(config_.hidden_dim))); - pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); - pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); - pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); - return gb->scalar_multiply(gb->add(pli_proj, pli_embed), 1.0f / std::sqrt(2.0f)); -} - -std::pair Gemma4Model::build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input) { - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - return {hidden, pli_combined}; -} - -void Gemma4Model::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens) { - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - gb->set_input(pli_input, input_data.data(), Precision::FP32); -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache) { - return forward_from_embeddings(gb, hidden, hidden, pli_tokens, seq_len, backend, use_cache); -} - -size_t Gemma4Model::build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend) { - if (config_.hidden_size_per_layer_input == 0) - return 0; - - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_combined = build_pli_combined(gb, hidden, pli_embed, seq_len, backend); - - std::vector pli_data(pli_tokens.size()); - for (size_t i = 0; i < pli_tokens.size(); i++) - pli_data[i] = static_cast(pli_tokens[i]); - gb->set_input(pli_input, pli_data.data(), Precision::FP32); - - return pli_combined; -} - -size_t Gemma4Model::forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache) { - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - if (config_.hidden_size_per_layer_input == 0) { - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - auto pli_combined = build_pli_combined_from_tokens(gb, pli_hidden_source, pli_tokens, seq_len, backend); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_combined, i, backend, use_cache, pos_offset); - - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (tokens.empty()) - throw std::runtime_error("Token sequence cannot be empty"); - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({tokens.size()}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, tokens.size(), backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -size_t Gemma4Model::forward_split(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - std::fill(shared_k_nodes_.begin(), shared_k_nodes_.end(), 0); - std::fill(shared_v_nodes_.begin(), shared_v_nodes_.end(), 0); - - size_t seq_len = tokens.size(); - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - if (config_.hidden_size_per_layer_input == 0) { - auto token_input = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, 0, i, backend, use_cache, shared_pos_offset); - - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - } - - size_t token_input, pli_input; - auto hidden_pli = build_preamble_and_embed(gb, seq_len, backend, token_input, pli_input); - size_t hidden = hidden_pli.first, pli = hidden_pli.second; - - for (uint32_t i = 0; i < first_shared_layer_; i++) - hidden = apply_transformer_layer(gb, hidden, pli, i, backend, use_cache, pos_offset); - - hidden = gb->index(hidden, seq_len - 1, 0); - hidden = gb->reshape(hidden, {1, config_.hidden_dim}); - auto pli_last = gb->index(pli, seq_len - 1, 0); - pli_last = gb->reshape(pli_last, {1, config_.num_layers * config_.hidden_size_per_layer_input}); - - size_t shared_pos_offset = pos_offset + seq_len - 1; - for (uint32_t i = first_shared_layer_; i < config_.num_layers; i++) - hidden = apply_transformer_layer(gb, hidden, pli_last, i, backend, use_cache, shared_pos_offset); - - set_token_inputs(gb, token_input, pli_input, tokens); - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - - -void Gemma4Model::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) - return; - - if (has_npu_prefill()) { - size_t npu_chunk_size = static_cast(npu_prefill_->get_chunk_size()); - if (tokens.size() > npu_chunk_size) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - } - - static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; - bool use_split = config_.num_kv_shared_layers > 0 - && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS - && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); - - if (!use_split) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - auto process_chunk = [&](const std::vector& chunk) { - forward_split(chunk, true); - gb->execute(profile_file); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - for (size_t i = 0; i < num_full_chunks; ++i) { - size_t start = i * chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); - if (i == 1) - gb->set_prefill_mode(true); - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - process_chunk(std::vector(tokens.begin() + final_start, tokens.end())); -} - -} -} diff --git a/cactus/models/gemma4/model_gemma4.h b/cactus/models/gemma4/model_gemma4.h deleted file mode 100644 index ce157decd..000000000 --- a/cactus/models/gemma4/model_gemma4.h +++ /dev/null @@ -1,413 +0,0 @@ -#pragma once - -#include "../model.h" - -bool test_gemma4_vision(bool expect_npu); -bool test_gemma4_audio(bool expect_npu); - -namespace cactus { -namespace engine { - -class Gemma4Model : public Model { - friend class Gemma4MmModel; -public: - Gemma4Model(); - explicit Gemma4Model(const Config& config); - ~Gemma4Model() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_moe(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - std::vector get_kv_layer_dims() const override; - std::vector get_kv_layer_heads() const override; - void compact_kv_cache() override; - - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend, bool use_cache); - size_t forward_from_embeddings(CactusGraph* gb, size_t hidden, size_t pli_hidden_source, - const std::vector& pli_tokens, size_t seq_len, - ComputeBackend backend, bool use_cache); - size_t build_pli_combined_from_tokens(CactusGraph* gb, size_t hidden, - const std::vector& pli_tokens, - size_t seq_len, ComputeBackend backend); - -private: - size_t forward_split(const std::vector& tokens, bool use_cache); - - std::pair build_preamble_and_embed(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input); - - void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens); - - size_t build_pli_combined(CactusGraph* gb, size_t hidden, size_t pli_embed, - size_t seq_len, ComputeBackend backend); - - size_t build_per_layer_input(CactusGraph* gb, size_t hidden, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend) const; - - bool is_global_layer(uint32_t idx) const; - size_t apply_partial_rope(CactusGraph* gb, size_t tensor, size_t head_dim, size_t rot_dim, - float rope_freq, size_t position_offset); - size_t apply_transformer_layer(CactusGraph* gb, size_t hidden, size_t pli, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t pos_offset); - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t embed_tokens_per_layer; - size_t per_layer_model_proj; - size_t per_layer_proj_norm; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - size_t per_layer_gate; - size_t per_layer_proj; - size_t post_per_layer_norm; - size_t layer_scalar; - - std::vector moe_w1_experts; - std::vector moe_w3_experts; - std::vector moe_w2_experts; - size_t moe_per_expert_scale = 0; - size_t router_proj = 0; - size_t router_scale = 0; - size_t post_ffn_norm_1 = 0; - size_t pre_ffn_norm_2 = 0; - size_t post_ffn_norm_2 = 0; - }; - - std::vector layers; - } weight_nodes_; - - uint32_t first_shared_layer_ = 0; - std::vector kv_share_map_; - std::vector shared_k_nodes_; - std::vector shared_v_nodes_; - - std::vector<__fp16> v_norm_ones_weight_; - size_t v_norm_ones_node_ = 0; - size_t v_norm_ones_global_node_ = 0; -}; - -class Gemma4VisionModel : public Model { - friend class Gemma4MmModel; -public: - struct PreprocessedImage { - std::vector pixel_values; - size_t height; - size_t width; - size_t patch_height; - size_t patch_width; - size_t num_patches; - }; - - Gemma4VisionModel(); - explicit Gemma4VisionModel(const Config& config); - ~Gemma4VisionModel() override = default; - - PreprocessedImage preprocess_image(const std::string& image_path); - size_t forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4VisionModel: use forward_vision() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4VisionModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4VisionModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; - -private: - size_t build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend); - size_t build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend); - size_t build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend); - size_t build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, ComputeBackend backend); - std::pair build_2d_rope_nodes(CactusGraph* gb, const PreprocessedImage& img, size_t max_patches); - size_t build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches); - - struct VisionWeightNodes { - size_t patch_input_proj = 0; - size_t position_table = 0; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t attn_q_norm = 0; - size_t attn_k_norm = 0; - size_t input_layernorm = 0; - size_t post_attention_layernorm = 0; - size_t pre_feedforward_layernorm = 0; - size_t post_feedforward_layernorm = 0; - size_t mlp_gate_proj = 0; - size_t mlp_up_proj = 0; - size_t mlp_down_proj = 0; - size_t layer_scalar = 0; - }; - - std::vector layers; - size_t embed_vision_proj = 0; - size_t post_proj_norm = 0; - } vision_weights_; - - std::vector<__fp16> vision_v_norm_ones_; - size_t vision_v_norm_ones_node_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - - friend bool ::test_gemma4_vision(bool); -}; - -class Gemma4AudioModel : public Model { - friend class Gemma4MmModel; -public: - Gemma4AudioModel(); - explicit Gemma4AudioModel(const Config& config); - ~Gemma4AudioModel() override = default; - - struct ConformerContext { - size_t timing_fp16 = 0; - size_t front_pad = 0; - size_t back_pad = 0; - size_t seq_len = 0; - }; - - size_t forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - size_t build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend); - - size_t build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend); - - ConformerContext build_conformer_context(CactusGraph* gb, size_t sscp_output); - - size_t build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend); - size_t build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - size_t build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend); - size_t build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend); - - struct AudioWeightNodes { - size_t sscp_conv0_weight = 0; - size_t sscp_conv0_norm = 0; - size_t sscp_conv1_weight = 0; - size_t sscp_conv1_norm = 0; - size_t sscp_input_proj = 0; - - struct ClipBounds { - float in_min = -1e10f, in_max = 1e10f; - float out_min = -1e10f, out_max = 1e10f; - }; - - struct ConformerLayerWeights { - size_t ffw_start_1 = 0, ffw_start_2 = 0; - ClipBounds ffw_start_1_clip, ffw_start_2_clip; - size_t ffw_start_pre_norm = 0, ffw_start_post_norm = 0; - size_t attn_q = 0, attn_k = 0, attn_v = 0; - ClipBounds attn_q_clip, attn_k_clip, attn_v_clip; - size_t attn_per_dim_scale = 0; - size_t attn_rel_pos_proj = 0; - size_t attn_post = 0; - ClipBounds attn_post_clip; - size_t attn_pre_norm = 0, attn_post_norm = 0; - size_t lconv_start = 0, lconv_depthwise = 0, lconv_end = 0; - ClipBounds lconv_start_clip, lconv_end_clip; - size_t lconv_pre_norm = 0, lconv_conv_norm = 0; - size_t ffw_end_1 = 0, ffw_end_2 = 0; - ClipBounds ffw_end_1_clip, ffw_end_2_clip; - size_t ffw_end_pre_norm = 0, ffw_end_post_norm = 0; - size_t block_norm = 0; - }; - std::vector layers; - - size_t output_proj = 0; - size_t output_proj_bias = 0; - - size_t embed_audio_proj = 0; - } audio_weights_; - - std::vector<__fp16> audio_proj_norm_ones_; - size_t audio_proj_norm_ones_node_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool disable_npu_ = false; - std::vector<__fp16> npu_audio_input_scratch_; - std::vector<__fp16> npu_audio_output_scratch_; - std::vector<__fp16> npu_audio_reorder_scratch_; - - friend bool ::test_gemma4_audio(bool); - -protected: - size_t forward(const std::vector&, bool) override { - throw std::runtime_error("Gemma4AudioModel: use forward_audio() instead"); - } - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_attention unused"); - } - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Gemma4AudioModel: build_mlp unused"); - } - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Gemma4AudioModel: build_transformer_block unused"); - } - void load_weights_to_graph(CactusGraph* gb) override; -}; - -class Gemma4MmModel : public Model { -public: - Gemma4MmModel(); - explicit Gemma4MmModel(const Config& config); - ~Gemma4MmModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - void prefill(const std::vector& tokens, size_t chunk_size = 256, - const std::string& profile_file = "") override; - - void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = "") override; - - uint32_t decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - - uint32_t decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - uint32_t decode_with_media( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector& audio_features, - float temperature = -1.0f, float top_p = -1.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f); - - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void compact_kv_cache() override; - void remove_thinking_tokens(const std::vector>& ranges) override; - - void set_tool_constraints(const std::vector& tools) override; - void clear_tool_constraints() override; - void update_tool_constraints(uint32_t token_id) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ForwardResult { - size_t final_hidden_node; - size_t seq_len; - }; - - ForwardResult forward_multimodal(CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache); - - uint32_t decode_multimodal(const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty); - -public: - struct MultimodalInputs { - size_t hidden_node = 0; - size_t pli_hidden_source_node = 0; - std::vector pli_tokens; - size_t seq_len = 0; - }; - - const Gemma4VisionModel& vision_encoder() const { return vision_encoder_; } - Gemma4VisionModel& vision_encoder() { return vision_encoder_; } - const Gemma4AudioModel& audio_encoder() const { return audio_encoder_; } - Gemma4AudioModel& audio_encoder() { return audio_encoder_; } - const Gemma4Model& language_model() const { return language_model_; } - Gemma4Model& language_model() { return language_model_; } - MultimodalInputs build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend); - -private: - Gemma4VisionModel vision_encoder_; - Gemma4AudioModel audio_encoder_; - Gemma4Model language_model_; -}; - -} -} diff --git a/cactus/models/gemma4/model_gemma4_audio.cpp b/cactus/models/gemma4/model_gemma4_audio.cpp deleted file mode 100644 index 9d7e7cab8..000000000 --- a/cactus/models/gemma4/model_gemma4_audio.cpp +++ /dev/null @@ -1,781 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static const float INV_LN2 = 1.0f / std::log(2.0f); -static const float GEMMA4_AUDIO_K_SCALE = std::log(1.0f + std::exp(1.0f)) / std::log(2.0f); -static const char* GEMMA4_AUDIO_NPU_INPUT_NAME = "x"; - -static size_t shape_elements(const std::vector& shape) { - if (shape.empty()) return 0; - size_t total = 1; - for (int d : shape) { - if (d <= 0) return 0; - total *= static_cast(d); - } - return total; -} - -static size_t infer_npu_audio_max_frames(const std::vector& shape, size_t mel_bins) { - if (shape.size() == 2) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - if (s1 == mel_bins) return s0; - if (s0 == mel_bins) return s1; - } else if (shape.size() == 3) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - if (s0 == 1 && s2 == mel_bins) return s1; // [1, T, F] - if (s0 == 1 && s1 == mel_bins) return s2; // [1, F, T] - if (s1 == mel_bins && s2 == 1) return s0; // [T, F, 1] - if (s0 == mel_bins && s2 == 1) return s1; // [F, T, 1] - } else if (shape.size() == 4) { - size_t s0 = static_cast(shape[0]); - size_t s1 = static_cast(shape[1]); - size_t s2 = static_cast(shape[2]); - size_t s3 = static_cast(shape[3]); - if (s0 == 1 && s1 == 1 && s3 == mel_bins) return s2; // [1, 1, T, F] - if (s0 == 1 && s1 == 1 && s2 == mel_bins) return s3; // [1, 1, F, T] - if (s0 == 1 && s1 == mel_bins && s3 == 1) return s2; // [1, F, T, 1] - if (s0 == 1 && s2 == mel_bins && s3 == 1) return s1; // [1, T, F, 1] - } - return 0; -} - -static bool pack_gemma4_audio_for_npu(const std::vector& mel_features, - size_t frames, - size_t mel_bins, - const std::vector& input_shape, - std::vector<__fp16>& packed) { - if (input_shape.empty()) return false; - const size_t total = shape_elements(input_shape); - if (total == 0) return false; - packed.resize(total); - std::fill(packed.begin(), packed.end(), static_cast<__fp16>(0.0f)); - - auto tm = [&](size_t t, size_t m) -> __fp16 { - return static_cast<__fp16>(mel_features[t * mel_bins + m]); - }; - - if (input_shape.size() == 4) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - const size_t s3 = static_cast(input_shape[3]); - if (s0 != 1) return false; - - if (s1 == 1 && s2 >= frames && s3 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s3) + m] = tm(t, m); - return true; - } - if (s1 == 1 && s2 == mel_bins && s3 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s3) + t] = tm(t, m); - return true; - } - if (s1 >= frames && s2 == mel_bins && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((t * s2 + m) * s3)] = tm(t, m); - return true; - } - if (s1 == mel_bins && s2 >= frames && s3 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[((m * s2 + t) * s3)] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 3) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - - if (s0 == 1 && s1 >= frames && s2 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s2 + m] = tm(t, m); - return true; - } - if (s0 == 1 && s1 == mel_bins && s2 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s2 + t] = tm(t, m); - return true; - } - if (s0 >= frames && s1 == mel_bins && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(t * s1 + m) * s2] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames && s2 == 1) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[(m * s1 + t) * s2] = tm(t, m); - return true; - } - return false; - } - - if (input_shape.size() == 2) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - - if (s0 >= frames && s1 == mel_bins) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[t * s1 + m] = tm(t, m); - return true; - } - if (s0 == mel_bins && s1 >= frames) { - for (size_t t = 0; t < frames; ++t) - for (size_t m = 0; m < mel_bins; ++m) - packed[m * s1 + t] = tm(t, m); - return true; - } - return false; - } - - return false; -} - -struct NPUAudioOutputLayout { - std::vector dims; - size_t hidden_axis = SIZE_MAX; - size_t hidden_dim = 0; - size_t time_steps = 0; -}; - -static bool infer_npu_audio_output_layout(const std::vector& output_shape, - size_t elements_written, - size_t expected_hidden_dim, - NPUAudioOutputLayout& layout) { - if (elements_written == 0) return false; - - layout = NPUAudioOutputLayout{}; - for (int d : output_shape) { - if (d > 0) layout.dims.push_back(static_cast(d)); - } - - if (layout.dims.empty()) { - if (expected_hidden_dim == 0 || (elements_written % expected_hidden_dim) != 0) { - return false; - } - layout.dims = {elements_written / expected_hidden_dim, expected_hidden_dim}; - } - - const size_t rank = layout.dims.size(); - size_t hidden_axis = SIZE_MAX; - if (expected_hidden_dim > 0) { - if (layout.dims[rank - 1] == expected_hidden_dim) { - hidden_axis = rank - 1; - } else if (rank >= 2 && layout.dims[rank - 2] == expected_hidden_dim) { - hidden_axis = rank - 2; - } else { - for (size_t i = 0; i < rank; ++i) { - if (layout.dims[i] == expected_hidden_dim) { - hidden_axis = i; - break; - } - } - } - } - - if (hidden_axis == SIZE_MAX) { - hidden_axis = rank - 1; - } - - const size_t hidden_dim = layout.dims[hidden_axis]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) { - return false; - } - - layout.hidden_axis = hidden_axis; - layout.hidden_dim = hidden_dim; - layout.time_steps = elements_written / hidden_dim; - return layout.time_steps > 0; -} - -static bool materialize_npu_audio_time_major(const __fp16* src, - const NPUAudioOutputLayout& layout, - size_t time_steps, - std::vector<__fp16>& dst) { - if (!src || layout.dims.empty() || layout.hidden_dim == 0 || layout.time_steps == 0) { - return false; - } - if (layout.hidden_axis >= layout.dims.size() || time_steps == 0 || time_steps > layout.time_steps) { - return false; - } - - const size_t rank = layout.dims.size(); - if (layout.hidden_axis == rank - 1) { - // Already [time, hidden] in row-major flattening. - return false; - } - - std::vector strides(rank, 1); - for (size_t i = rank; i-- > 1;) { - strides[i - 1] = strides[i] * layout.dims[i]; - } - - std::vector non_hidden_axes; - non_hidden_axes.reserve(rank - 1); - for (size_t i = 0; i < rank; ++i) { - if (i != layout.hidden_axis) non_hidden_axes.push_back(i); - } - - dst.assign(time_steps * layout.hidden_dim, static_cast<__fp16>(0.0f)); - - for (size_t t = 0; t < time_steps; ++t) { - size_t rem = t; - size_t base_offset = 0; - for (size_t i = non_hidden_axes.size(); i-- > 0;) { - size_t axis = non_hidden_axes[i]; - size_t dim = layout.dims[axis]; - size_t coord = rem % dim; - rem /= dim; - base_offset += coord * strides[axis]; - } - - for (size_t h = 0; h < layout.hidden_dim; ++h) { - size_t src_index = base_offset + h * strides[layout.hidden_axis]; - dst[t * layout.hidden_dim + h] = src[src_index]; - } - } - - return true; -} - -static size_t graph_clamp(CactusGraph* gb, size_t x, float lo, float hi) { - if (lo >= hi || std::isinf(lo) || std::isinf(hi)) return x; - x = gb->scalar_add(x, -lo); - x = gb->relu(x); - x = gb->scalar_add(x, lo); - x = gb->scalar_multiply(gb->scalar_add(x, -hi), -1.0f); - x = gb->relu(x); - x = gb->scalar_add(gb->scalar_multiply(x, -1.0f), hi); - return x; -} - -static size_t clipped_matmul(CactusGraph* gb, size_t input, size_t weight, - float in_min, float in_max, float out_min, float out_max, - ComputeBackend backend) { - size_t clamped_in = graph_clamp(gb, input, in_min, in_max); - size_t result = gb->matmul(clamped_in, weight, true, backend); - return graph_clamp(gb, result, out_min, out_max); -} - -static float read_scalar_weight(CactusGraph* gb, const std::string& path) { - size_t node = gb->mmap_weights(path); - const auto& buf = gb->get_output_buffer(node); - if (buf.precision == Precision::FP16) - return static_cast(*buf.data_as<__fp16>()); - if (buf.precision == Precision::FP32) - return *buf.data_as(); - return 0.0f; -} - -static Gemma4AudioModel::AudioWeightNodes::ClipBounds load_clip_bounds(CactusGraph* gb, const std::string& prefix) { - Gemma4AudioModel::AudioWeightNodes::ClipBounds cb{0, 0, 0, 0}; - std::string in_min_path = prefix + "_input_min.weights"; - if (!std::filesystem::exists(in_min_path)) - return cb; - cb.in_min = read_scalar_weight(gb, in_min_path); - cb.in_max = read_scalar_weight(gb, prefix + "_input_max.weights"); - cb.out_min = read_scalar_weight(gb, prefix + "_output_min.weights"); - cb.out_max = read_scalar_weight(gb, prefix + "_output_max.weights"); - return cb; -} - -static void get_conv2d_output_hw(const CactusGraph* gb, size_t node, size_t& h, size_t& w) { - const auto& buf = gb->get_output_buffer(node); - h = buf.shape[2]; - w = buf.shape[3]; -} - -Gemma4AudioModel::Gemma4AudioModel() : Model() {} -Gemma4AudioModel::Gemma4AudioModel(const Config& config) : Model(config) {} - -void Gemma4AudioModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - auto resolve_existing = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> std::string { - std::string preferred_path = resolve(preferred); - if (std::filesystem::exists(preferred_path)) { - return preferred_path; - } - if (!legacy.empty()) { - std::string legacy_path = resolve(legacy); - if (std::filesystem::exists(legacy_path)) { - return legacy_path; - } - } - return preferred_path; - }; - auto load_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) -> size_t { - return gb->mmap_weights(resolve_existing(preferred, legacy)); - }; - auto load_clip_from_weight = [&](const std::string& preferred, - const std::string& legacy = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string weight_path = resolve_existing(preferred, legacy); - static const std::string suffix = ".weights"; - if (weight_path.size() > suffix.size() && - weight_path.compare(weight_path.size() - suffix.size(), suffix.size(), suffix) == 0) { - return load_clip_bounds(gb, weight_path.substr(0, weight_path.size() - suffix.size())); - } - return Gemma4AudioModel::AudioWeightNodes::ClipBounds{0, 0, 0, 0}; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/audio_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - npu_encoder_->preallocate(input_shape, GEMMA4_AUDIO_NPU_INPUT_NAME, ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_WARN("npu", "[gemma4-audio] found audio_encoder.mlpackage but failed to enable NPU audio encoder; using CPU"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4-audio] audio_encoder.mlpackage not found; using CPU audio encoder"); - } - } else if (!disable_npu_) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU backend unavailable on this device; using CPU audio encoder"); - } - - // Keep CPU audio weights available even when the NPU encoder is enabled. - // The runtime may still fall back to the graph path for unsupported inputs. - { - audio_weights_.sscp_conv0_weight = load_weight( - "audio_subsample_conv_projection_conv_0_conv.weights", - "audio_subsample_conv_projection_layer0_conv.weights"); - audio_weights_.sscp_conv0_norm = load_weight( - "audio_subsample_conv_projection_conv_0_norm.weights", - "audio_subsample_conv_projection_layer0_norm.weights"); - audio_weights_.sscp_conv1_weight = load_weight( - "audio_subsample_conv_projection_conv_1_conv.weights", - "audio_subsample_conv_projection_layer1_conv.weights"); - audio_weights_.sscp_conv1_norm = load_weight( - "audio_subsample_conv_projection_conv_1_norm.weights", - "audio_subsample_conv_projection_layer1_norm.weights"); - audio_weights_.sscp_input_proj = load_weight( - "audio_subsample_conv_projection_input_proj.weights", - "audio_subsample_conv_projection_input_proj_linear.weights"); - - audio_weights_.layers.resize(config_.audio_num_layers); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) { - auto& layer = audio_weights_.layers[i]; - - auto layer_weight = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) -> size_t { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - auto layer_clip = [&](const std::string& preferred_suffix, - const std::string& legacy_suffix = std::string()) - -> Gemma4AudioModel::AudioWeightNodes::ClipBounds { - std::string legacy = legacy_suffix.empty() ? preferred_suffix : legacy_suffix; - return load_clip_from_weight( - "audio_conformer_" + std::to_string(i) + "_" + preferred_suffix, - "audio_layers_" + std::to_string(i) + "_" + legacy); - }; - - layer.ffw_start_1 = layer_weight("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2 = layer_weight("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_1_clip = layer_clip("ffw_layer_start_ffw_layer_1.weights", "feed_forward1_ffw_layer_1.weights"); - layer.ffw_start_2_clip = layer_clip("ffw_layer_start_ffw_layer_2.weights", "feed_forward1_ffw_layer_2.weights"); - layer.ffw_start_pre_norm = layer_weight("ffw_layer_start_pre_layer_norm.weights", "feed_forward1_pre_layer_norm.weights"); - layer.ffw_start_post_norm = layer_weight("ffw_layer_start_post_layer_norm.weights", "feed_forward1_post_layer_norm.weights"); - - layer.attn_q = layer_weight("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k = layer_weight("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v = layer_weight("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_q_clip = layer_clip("attention_attn_q_proj.weights", "self_attn_q_proj.weights"); - layer.attn_k_clip = layer_clip("attention_attn_k_proj.weights", "self_attn_k_proj.weights"); - layer.attn_v_clip = layer_clip("attention_attn_v_proj.weights", "self_attn_v_proj.weights"); - layer.attn_per_dim_scale = layer_weight("attention_attn_per_dim_scale.weights", "self_attn_per_dim_scale.weights"); - layer.attn_rel_pos_proj = layer_weight( - "attention_attn_relative_position_embedding_pos_proj.weights", - "self_attn_relative_k_proj.weights"); - layer.attn_post = layer_weight("attention_post.weights", "self_attn_post.weights"); - layer.attn_post_clip = layer_clip("attention_post.weights", "self_attn_post.weights"); - layer.attn_pre_norm = layer_weight("attention_pre_attn_norm.weights", "norm_pre_attn.weights"); - layer.attn_post_norm = layer_weight("attention_post_norm.weights", "norm_post_attn.weights"); - - layer.lconv_start = layer_weight("lconv1d_linear_start.weights"); - layer.lconv_depthwise = layer_weight("lconv1d_depthwise_conv1d.weights"); - layer.lconv_end = layer_weight("lconv1d_linear_end.weights"); - layer.lconv_start_clip = layer_clip("lconv1d_linear_start.weights"); - layer.lconv_end_clip = layer_clip("lconv1d_linear_end.weights"); - layer.lconv_pre_norm = layer_weight("lconv1d_pre_layer_norm.weights"); - layer.lconv_conv_norm = layer_weight("lconv1d_conv_norm.weights"); - - layer.ffw_end_1 = layer_weight("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2 = layer_weight("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_1_clip = layer_clip("ffw_layer_end_ffw_layer_1.weights", "feed_forward2_ffw_layer_1.weights"); - layer.ffw_end_2_clip = layer_clip("ffw_layer_end_ffw_layer_2.weights", "feed_forward2_ffw_layer_2.weights"); - layer.ffw_end_pre_norm = layer_weight("ffw_layer_end_pre_layer_norm.weights", "feed_forward2_pre_layer_norm.weights"); - layer.ffw_end_post_norm = layer_weight("ffw_layer_end_post_layer_norm.weights", "feed_forward2_post_layer_norm.weights"); - - layer.block_norm = layer_weight("norm.weights", "norm_out.weights"); - } - - audio_weights_.output_proj = load_weight("audio_output_proj.weights"); - audio_weights_.output_proj_bias = load_weight("audio_output_proj.bias"); - } - audio_weights_.embed_audio_proj = load_weight("embed_audio_proj.weights"); - - size_t proj_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - audio_proj_norm_ones_.assign(proj_out_dim, static_cast<__fp16>(1.0f)); - audio_proj_norm_ones_node_ = gb->input({proj_out_dim}, Precision::FP16); - gb->set_external_input(audio_proj_norm_ones_node_, audio_proj_norm_ones_.data(), Precision::FP16); - - output_weight_node_id_ = 0; -} - -size_t Gemma4AudioModel::build_sscp(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - size_t mel_bins = config_.audio_input_feat_size; - size_t conv0_ch = config_.audio_sscp_conv0_channels; - size_t conv1_ch = config_.audio_sscp_conv1_channels; - - size_t mel_input = gb->input({num_frames, mel_bins}, Precision::FP32); - gb->set_input(mel_input, mel_features.data(), Precision::FP32); - size_t mel_fp16 = gb->precision_cast(mel_input, Precision::FP16); - - size_t x = gb->reshape(mel_fp16, {1, 1, num_frames, mel_bins}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv0_weight); - size_t h1, w1; - get_conv2d_output_hw(gb, x, h1, w1); - - x = gb->reshape(x, {conv0_ch, h1, w1}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h1 * w1, conv0_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv0_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h1, w1, conv0_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - x = gb->reshape(x, {1, conv0_ch, h1, w1}); - - x = gb->conv2d_k3s2p1(x, audio_weights_.sscp_conv1_weight); - size_t h2, w2; - get_conv2d_output_hw(gb, x, h2, w2); - - x = gb->reshape(x, {conv1_ch, h2, w2}); - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2 * w2, conv1_ch}); - x = gb->layernorm(x, audio_weights_.sscp_conv1_norm, config_.audio_sscp_conv_eps); - x = gb->reshape(x, {h2, w2, conv1_ch}); - x = gb->transposeN(x, {2, 0, 1}); - x = gb->relu(x); - - x = gb->transposeN(x, {1, 2, 0}); - x = gb->reshape(x, {h2, w2 * conv1_ch}); - - return gb->matmul(x, audio_weights_.sscp_input_proj, true, backend); -} - -static std::vector compute_timing_signal(size_t num_positions, size_t hidden_dim, size_t max_past) { - size_t num_timescales = hidden_dim / 2; - float log_ts_inc = std::log(1.0e4f) / std::max(static_cast(num_timescales) - 1, 1); - - std::vector signal(num_positions * hidden_dim); - for (size_t p = 0; p < num_positions; p++) { - float pos = static_cast(max_past) - static_cast(p); - for (size_t i = 0; i < num_timescales; i++) { - float scaled = pos * std::exp(-static_cast(i) * log_ts_inc); - signal[p * hidden_dim + i] = std::sin(scaled); - signal[p * hidden_dim + num_timescales + i] = std::cos(scaled); - } - } - return signal; -} - -static size_t create_zero_pad_node(CactusGraph* gb, size_t time_len, size_t num_heads, size_t head_dim) { - if (time_len == 0) return 0; - std::vector<__fp16> zeros(time_len * num_heads * head_dim, static_cast<__fp16>(0.0f)); - size_t node = gb->input({time_len, num_heads, head_dim}, Precision::FP16); - gb->set_input(node, zeros.data(), Precision::FP16); - return node; -} - -Gemma4AudioModel::ConformerContext Gemma4AudioModel::build_conformer_context( - CactusGraph* gb, size_t sscp_output) { - - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = gb->get_output_buffer(sscp_output).shape[0]; - - auto timing_signal = compute_timing_signal(num_positions, hidden_dim, max_past); - size_t timing_input = gb->input({num_positions, hidden_dim}, Precision::FP32); - gb->set_input(timing_input, timing_signal.data(), Precision::FP32); - size_t timing_fp16 = gb->precision_cast(timing_input, Precision::FP16); - - size_t front_pad_len = (seq_len >= num_positions) ? seq_len - num_positions : 0; - size_t back_pad_len = seq_len > 0 ? seq_len - 1 : 0; - - return { - timing_fp16, - create_zero_pad_node(gb, front_pad_len, num_heads, head_dim), - create_zero_pad_node(gb, back_pad_len, num_heads, head_dim), - seq_len - }; -} - -size_t Gemma4AudioModel::build_conformer_ffw(CactusGraph* gb, size_t input, uint32_t layer_idx, - bool is_end, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float residual_weight = config_.audio_residual_weight; - float eps = config_.audio_rms_norm_eps; - - size_t residual = input; - - size_t pre_norm = is_end ? layer.ffw_end_pre_norm : layer.ffw_start_pre_norm; - size_t post_norm = is_end ? layer.ffw_end_post_norm : layer.ffw_start_post_norm; - size_t w1 = is_end ? layer.ffw_end_1 : layer.ffw_start_1; - size_t w2 = is_end ? layer.ffw_end_2 : layer.ffw_start_2; - - auto& c1 = is_end ? layer.ffw_end_1_clip : layer.ffw_start_1_clip; - auto& c2 = is_end ? layer.ffw_end_2_clip : layer.ffw_start_2_clip; - - size_t x = gb->rms_norm(input, pre_norm, eps); - - x = clipped_matmul(gb, x, w1, c1.in_min, c1.in_max, c1.out_min, c1.out_max, backend); - x = gb->silu(x); - x = clipped_matmul(gb, x, w2, c2.in_min, c2.in_max, c2.out_min, c2.out_max, backend); - - x = gb->rms_norm(x, post_norm, eps); - - x = gb->scalar_multiply(x, residual_weight); - return gb->add(residual, x); -} - -size_t Gemma4AudioModel::build_conformer_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t num_heads = config_.audio_num_heads; - size_t head_dim = config_.audio_head_dim; - size_t hidden_dim = num_heads * head_dim; - float logit_cap = config_.audio_logit_cap; - size_t max_past = config_.audio_context_left > 0 ? config_.audio_context_left - 1 : 0; - size_t num_positions = max_past + config_.audio_context_right + 1; - size_t seq_len = ctx.seq_len; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.attn_pre_norm, eps); - - size_t q = clipped_matmul(gb, x, layer.attn_q, layer.attn_q_clip.in_min, layer.attn_q_clip.in_max, layer.attn_q_clip.out_min, layer.attn_q_clip.out_max, backend); - size_t k = clipped_matmul(gb, x, layer.attn_k, layer.attn_k_clip.in_min, layer.attn_k_clip.in_max, layer.attn_k_clip.out_min, layer.attn_k_clip.out_max, backend); - size_t v = clipped_matmul(gb, x, layer.attn_v, layer.attn_v_clip.in_min, layer.attn_v_clip.in_max, layer.attn_v_clip.out_min, layer.attn_v_clip.out_max, backend); - - float q_scale = (1.0f / std::sqrt(static_cast(head_dim))) * INV_LN2; - size_t q_dim_scale_fp16 = gb->precision_cast(layer.attn_per_dim_scale, Precision::FP16); - size_t q_dim_scale = gb->scalar_exp(q_dim_scale_fp16); - q_dim_scale = gb->scalar_add(q_dim_scale, 1.0f); - q_dim_scale = gb->scalar_log(q_dim_scale); - q_dim_scale = gb->scalar_multiply(q_dim_scale, q_scale); - - size_t q_flat = gb->reshape(q, {seq_len * num_heads, head_dim}); - q_flat = gb->multiply(q_flat, q_dim_scale); - q = gb->reshape(q_flat, {seq_len, hidden_dim}); - - k = gb->scalar_multiply(k, GEMMA4_AUDIO_K_SCALE); - - size_t sin_emb = gb->matmul(ctx.timing_fp16, layer.attn_rel_pos_proj, true, backend); - size_t sin_emb_3d = gb->reshape(sin_emb, {num_positions, num_heads, head_dim}); - size_t R = 2 * seq_len - 1; - - size_t rel_key; - if (seq_len >= num_positions) { - rel_key = (ctx.front_pad != 0) ? gb->concat(ctx.front_pad, sin_emb_3d, 0) : sin_emb_3d; - if (ctx.back_pad != 0) - rel_key = gb->concat(rel_key, ctx.back_pad, 0); - } else { - size_t start_idx = num_positions - seq_len; - size_t sin_emb_sliced = gb->slice(sin_emb_3d, 0, start_idx, seq_len); - rel_key = (ctx.back_pad != 0) ? gb->concat(sin_emb_sliced, ctx.back_pad, 0) : sin_emb_sliced; - } - - size_t rel_key_4d = gb->reshape(rel_key, {1, R, num_heads, head_dim}); - size_t q4 = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q4, rel_key_4d, 1.0f); - - size_t k4 = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - size_t attn = gb->attention_masked(q4, k4, v4, rel_bias, 1.0f, - true, backend, true, 0, max_past, logit_cap); - - size_t attn_out = gb->reshape(attn, {seq_len, hidden_dim}); - - size_t out = clipped_matmul(gb, attn_out, layer.attn_post, layer.attn_post_clip.in_min, layer.attn_post_clip.in_max, layer.attn_post_clip.out_min, layer.attn_post_clip.out_max, backend); - out = gb->rms_norm(out, layer.attn_post_norm, eps); - - return gb->add(residual, out); -} - -size_t Gemma4AudioModel::build_conformer_lconv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = audio_weights_.layers[layer_idx]; - float eps = config_.audio_rms_norm_eps; - size_t kernel_size = config_.audio_conf_conv_kernel_size; - size_t hidden_dim = config_.audio_hidden_dim; - - size_t residual = input; - - size_t x = gb->rms_norm(input, layer.lconv_pre_norm, eps); - - x = clipped_matmul(gb, x, layer.lconv_start, layer.lconv_start_clip.in_min, layer.lconv_start_clip.in_max, layer.lconv_start_clip.out_min, layer.lconv_start_clip.out_max, backend); - - x = gb->glu(x, -1); - - const auto& x_buf = gb->get_output_buffer(x); - size_t seq_len = x_buf.shape[0]; - x = gb->reshape(x, {1, seq_len, hidden_dim}); - - x = gb->conv1d_causal(x, layer.lconv_depthwise, kernel_size); - - x = gb->reshape(x, {seq_len, hidden_dim}); - - x = gb->rms_norm(x, layer.lconv_conv_norm, eps); - x = gb->silu(x); - - x = clipped_matmul(gb, x, layer.lconv_end, layer.lconv_end_clip.in_min, layer.lconv_end_clip.in_max, layer.lconv_end_clip.out_min, layer.lconv_end_clip.out_max, backend); - - return gb->add(x, residual); -} - -size_t Gemma4AudioModel::build_conformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - const ConformerContext& ctx, ComputeBackend backend) { - hidden = build_conformer_ffw(gb, hidden, layer_idx, false, backend); - hidden = build_conformer_attention(gb, hidden, layer_idx, ctx, backend); - hidden = build_conformer_lconv1d(gb, hidden, layer_idx, backend); - hidden = build_conformer_ffw(gb, hidden, layer_idx, true, backend); - return gb->rms_norm(hidden, audio_weights_.layers[layer_idx].block_norm, config_.audio_rms_norm_eps); -} - -size_t Gemma4AudioModel::forward_audio(CactusGraph* gb, const std::vector& mel_features, - size_t num_frames, ComputeBackend backend) { - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - size_t mel_bins = static_cast(config_.audio_input_feat_size); - size_t max_npu_frames = infer_npu_audio_max_frames(npu_input_shape, mel_bins); - size_t copy_frames = max_npu_frames > 0 ? std::min(num_frames, max_npu_frames) : num_frames; - size_t input_values = copy_frames * mel_bins; - - if (mel_features.size() < input_values) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] insufficient mel input values; falling back to CPU audio encoder"); - } else if (!pack_gemma4_audio_for_npu(mel_features, copy_frames, mel_bins, npu_input_shape, - npu_audio_input_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] unsupported NPU input shape; falling back to CPU audio encoder"); - } else { - size_t t1 = (copy_frames + 1) / 2; - size_t t2 = (t1 + 1) / 2; - size_t expected_out_dim = config_.audio_output_proj_dims > 0 - ? static_cast(config_.audio_output_proj_dims) - : static_cast(config_.audio_hidden_dim); - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = npu_encoder_->get_output_buffer_size(); - if (out_elements == 0) out_elements = shape_elements(out_shape); - if (out_elements == 0) out_elements = std::max(1, t2 * expected_out_dim); - - if (npu_audio_output_scratch_.size() < out_elements) - npu_audio_output_scratch_.resize(out_elements); - - size_t written = npu_encoder_->encode( - npu_audio_input_scratch_.data(), npu_audio_output_scratch_.data(), npu_input_shape, - GEMMA4_AUDIO_NPU_INPUT_NAME, ""); - - NPUAudioOutputLayout out_layout; - if (written > 0 && - infer_npu_audio_output_layout(out_shape, written, expected_out_dim, out_layout) && - out_layout.hidden_dim == expected_out_dim) { - const __fp16* src = npu_audio_output_scratch_.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - size_t cached_count = npu_encoder_->get_output_buffer_size(); - size_t required = out_layout.time_steps * out_layout.hidden_dim; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - size_t actual_time = std::min(t2, out_layout.time_steps); - if (actual_time > 0) { - const __fp16* final_src = src; - if (out_layout.hidden_axis != (out_layout.dims.size() - 1)) { - if (!materialize_npu_audio_time_major(src, out_layout, actual_time, - npu_audio_reorder_scratch_)) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] failed to reorder NPU output layout; falling back to CPU audio encoder"); - } else { - final_src = npu_audio_reorder_scratch_.data(); - } - } - - if (final_src != nullptr) { - size_t hidden = gb->input({actual_time, out_layout.hidden_dim}, Precision::FP16); - gb->set_input(hidden, final_src, Precision::FP16); - return hidden; - } - } - } - - if (written > 0 && out_layout.hidden_dim != expected_out_dim) { - CACTUS_LOG_WARN("npu", "[gemma4-audio] NPU output dim mismatch; falling back to CPU audio encoder"); - } else { - CACTUS_LOG_WARN("npu", "audio encode failed, falling back to CPU"); - } - } - } - - size_t hidden = build_sscp(gb, mel_features, num_frames, backend); - ConformerContext ctx = build_conformer_context(gb, hidden); - for (uint32_t i = 0; i < config_.audio_num_layers; i++) - hidden = build_conformer_block(gb, hidden, i, ctx, backend); - - if (config_.audio_output_proj_dims > 0) { - hidden = gb->matmul(hidden, audio_weights_.output_proj, true, backend); - hidden = gb->add(hidden, audio_weights_.output_proj_bias); - } - - return hidden; -} - -size_t Gemma4AudioModel::build_audio_projector(CactusGraph* gb, size_t audio_features, ComputeBackend backend) { - size_t normed = gb->rms_norm(audio_features, audio_proj_norm_ones_node_, config_.audio_rms_norm_eps); - size_t projected = gb->matmul(normed, audio_weights_.embed_audio_proj, true, backend); - return gb->scalar_multiply(projected, 1.0f / 16.0f); -} - -} -} diff --git a/cactus/models/gemma4/model_gemma4_mm.cpp b/cactus/models/gemma4/model_gemma4_mm.cpp deleted file mode 100644 index ab825dfb8..000000000 --- a/cactus/models/gemma4/model_gemma4_mm.cpp +++ /dev/null @@ -1,473 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -Gemma4MmModel::Gemma4MmModel() : Model() { - config_.model_type = Config::ModelType::GEMMA4; -} - -Gemma4MmModel::Gemma4MmModel(const Config& config) - : Model(config), vision_encoder_(config), audio_encoder_(config), language_model_(config) {} - -bool Gemma4MmModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, false)) - return false; - - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) - throw std::runtime_error("Shared graph was not initialized for Gemma4MmModel"); - - bool has_vision = config_.vision_num_layers > 0 || config_.vision_embed_dim > 0; - bool has_audio = config_.audio_num_layers > 0 || config_.audio_hidden_dim > 0; - - if (has_vision) { - if (!vision_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize vision encoder"); - } - - if (has_audio) { - if (!audio_encoder_.init(shared_graph, model_folder, context_size, "", false)) - throw std::runtime_error("Failed to initialize audio encoder"); - } - - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) - throw std::runtime_error("Failed to initialize language model"); - - output_weight_node_id_ = language_model_.output_weight_node_id_; - - if (do_warmup) { - std::vector warmup_tokens = {2}; - language_model_.forward(warmup_tokens); - auto* gb2 = static_cast(language_model_.graph_handle_); - gb2->execute(); - language_model_.reset_cache(); - } - - return true; -} - -void Gemma4MmModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); -} - -void Gemma4MmModel::compact_kv_cache() { - language_model_.compact_kv_cache(); -} - -void Gemma4MmModel::remove_thinking_tokens(const std::vector>& ranges) { - language_model_.remove_thinking_tokens(ranges); -} - -void Gemma4MmModel::set_tool_constraints(const std::vector& tools) { - language_model_.set_tool_constraints(tools); -} - -void Gemma4MmModel::clear_tool_constraints() { - language_model_.clear_tool_constraints(); -} - -void Gemma4MmModel::update_tool_constraints(uint32_t token_id) { - language_model_.update_tool_constraints(token_id); -} - -void Gemma4MmModel::load_weights_to_graph(CactusGraph*) { - output_weight_node_id_ = 0; -} - -Gemma4MmModel::ForwardResult Gemma4MmModel::forward_multimodal( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend, bool use_cache) { - - auto inputs = build_multimodal_inputs( - gb, tokens, image_paths, audio_features, audio_num_frames, backend); - - size_t final_hidden = language_model_.forward_from_embeddings( - gb, - inputs.hidden_node, - inputs.pli_hidden_source_node, - inputs.pli_tokens, - inputs.seq_len, - backend, - use_cache); - - return ForwardResult{final_hidden, inputs.seq_len}; -} - -Gemma4MmModel::MultimodalInputs Gemma4MmModel::build_multimodal_inputs( - CactusGraph* gb, const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - ComputeBackend backend) { - - size_t vision_soft_node = 0; - size_t num_vision_soft_tokens = 0; - size_t audio_soft_node = 0; - size_t num_audio_soft_tokens = 0; - - if (!image_paths.empty()) { - auto preprocessed = vision_encoder_.preprocess_image(image_paths[0]); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - vision_soft_node = vision_encoder_.build_vision_projector(gb, vision_output, backend); - uint32_t k = config_.vision_pooling_kernel_size; - num_vision_soft_tokens = (preprocessed.patch_width / k) * (preprocessed.patch_height / k); - } - - if (audio_features && !audio_features->empty()) { - size_t audio_output = audio_encoder_.forward_audio(gb, *audio_features, audio_num_frames, backend); - audio_soft_node = audio_encoder_.build_audio_projector(gb, audio_output, backend); - const auto& audio_buf = gb->get_output_buffer(audio_soft_node); - num_audio_soft_tokens = audio_buf.shape[0]; - } - - uint32_t image_token_id = config_.image_token_id; - uint32_t audio_token_id = config_.audio_token_id; - uint32_t pad_token_id = config_.pad_token_id; - - std::vector sequence_nodes; - std::vector current_text; - std::vector pli_tokens; - size_t total_seq_len = 0; - size_t vision_offset = 0; - size_t audio_offset = 0; - - auto flush_text = [&]() { - if (current_text.empty()) return; - size_t seg_len = current_text.size(); - size_t input_node = gb->input({seg_len}, Precision::FP32); - - auto hidden = gb->scalar_multiply( - gb->embedding(language_model_.embedding_node_id_, input_node), - std::sqrt(static_cast(config_.hidden_dim))); - - std::vector input_data(seg_len); - for (size_t i = 0; i < seg_len; i++) - input_data[i] = static_cast(current_text[i]); - gb->set_input(input_node, input_data.data(), Precision::FP32); - - sequence_nodes.push_back(hidden); - for (auto t : current_text) - pli_tokens.push_back(t); - total_seq_len += seg_len; - current_text.clear(); - }; - - auto append_soft_region = [&](size_t soft_node, size_t& soft_offset, size_t total_soft_tokens, - size_t placeholder_count) { - size_t to_insert = std::min(placeholder_count, total_soft_tokens - soft_offset); - if (to_insert > 0) { - sequence_nodes.push_back(gb->slice(soft_node, 0, soft_offset, to_insert)); - for (size_t j = 0; j < to_insert; j++) - pli_tokens.push_back(pad_token_id); - total_seq_len += to_insert; - soft_offset += to_insert; - } - }; - - auto flush_vision_region = [&](size_t placeholder_count) { - append_soft_region(vision_soft_node, vision_offset, num_vision_soft_tokens, placeholder_count); - }; - - auto flush_audio_region = [&](size_t placeholder_count) { - append_soft_region(audio_soft_node, audio_offset, num_audio_soft_tokens, placeholder_count); - }; - - bool in_image_region = false; - bool in_audio_region = false; - size_t region_count = 0; - - for (size_t i = 0; i < tokens.size(); i++) { - uint32_t tok = tokens[i]; - bool is_vision_token = (tok == image_token_id && image_token_id != 0); - bool is_audio_token = (tok == audio_token_id && audio_token_id != 0); - - if (is_vision_token) { - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - if (!in_image_region) { - flush_text(); - in_image_region = true; - region_count = 0; - } - region_count++; - } else if (is_audio_token) { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (!in_audio_region) { - flush_text(); - in_audio_region = true; - region_count = 0; - } - region_count++; - } else { - if (in_image_region) { - flush_vision_region(region_count); - in_image_region = false; - } - if (in_audio_region) { - flush_audio_region(region_count); - in_audio_region = false; - } - current_text.push_back(tok); - } - } - - if (in_image_region) - flush_vision_region(region_count); - if (in_audio_region) - flush_audio_region(region_count); - flush_text(); - - if (sequence_nodes.empty()) - throw std::runtime_error("No embedding nodes built"); - - size_t merged = sequence_nodes[0]; - for (size_t i = 1; i < sequence_nodes.size(); i++) - merged = gb->concat(merged, sequence_nodes[i], 0); - - return MultimodalInputs{ - .hidden_node = merged, - .pli_hidden_source_node = merged, - .pli_tokens = std::move(pli_tokens), - .seq_len = total_seq_len, - }; -} - -uint32_t Gemma4MmModel::decode_multimodal( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector* audio_features, - size_t audio_num_frames, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - bool has_media = !image_paths.empty() || (audio_features && !audio_features->empty()); - - if (!has_media) { - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) temperature = config_.default_temperature; - if (top_p < 0) top_p = config_.default_top_p; - if (top_k == 0) top_k = config_.default_top_k; - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t cached_len = language_model_.kv_cache_.current_seq_len; - if (cached_len >= tokens.size()) { - reset_cache(); - cached_len = 0; - } - std::vector forward_tokens = cached_len == 0 - ? tokens - : std::vector(tokens.end() - (tokens.size() - cached_len), tokens.end()); - - bool delta_has_media = std::any_of(forward_tokens.begin(), forward_tokens.end(), [&](uint32_t t) { - return (config_.image_token_id != 0 && t == config_.image_token_id) || - (config_.audio_token_id != 0 && t == config_.audio_token_id); - }); - - size_t final_hidden_node = 0; - size_t seq_len_for_updates = 0; - if (delta_has_media) { - auto result = forward_multimodal(gb, forward_tokens, image_paths, audio_features, - audio_num_frames, backend, true); - final_hidden_node = result.final_hidden_node; - seq_len_for_updates = result.seq_len; - } else { - final_hidden_node = language_model_.forward(forward_tokens, true); - seq_len_for_updates = forward_tokens.size(); - } - - auto last_hidden = gb->index(final_hidden_node, seq_len_for_updates - 1, 0); - const auto& last_buf = gb->get_output_buffer(last_hidden); - last_hidden = gb->reshape(last_hidden, {1, last_buf.shape[0]}); - - auto logits_node = gb->matmul(last_hidden, language_model_.output_weight_node_id_, true, backend); - - if (config_.final_logit_softcapping > 0.0f) { - float inv_cap = 1.0f / config_.final_logit_softcapping; - logits_node = gb->scalar_multiply(logits_node, inv_cap); - logits_node = gb->tanh(logits_node); - logits_node = gb->scalar_multiply(logits_node, config_.final_logit_softcapping); - } - - size_t sampled_token = - language_model_.sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - compute_entropy(gb, logits_node, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.update_kv_cache(gb, seq_len_for_updates); - - auto* output_ptr = gb->get_output(sampled_token); - uint32_t result_token = *static_cast(output_ptr); - language_model_.record_sampled_token(result_token); - return result_token; -} - -size_t Gemma4MmModel::forward(const std::vector& tokens, bool use_cache) { - return language_model_.forward(tokens, use_cache); -} - -uint32_t Gemma4MmModel::decode(const std::vector& tokens, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Gemma4MmModel::prefill(const std::vector& tokens, size_t chunk_size, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Gemma4MmModel::prefill_with_images(const std::vector& tokens, - const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto result = forward_multimodal(gb, tokens, image_paths, nullptr, 0, backend, true); - - if (!profile_file.empty()) - gb->execute(profile_file); - else - gb->execute(); - - language_model_.post_execute_updates(gb, result.seq_len); - language_model_.update_kv_cache(gb, result.seq_len); -} - -uint32_t Gemma4MmModel::decode_with_images( - const std::vector& tokens, const std::vector& image_paths, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - return decode_multimodal(tokens, image_paths, nullptr, 0, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -uint32_t Gemma4MmModel::decode_with_audio( - const std::vector& tokens, const std::vector& audio_features, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty, - float* /*out_token_time_start*/, float* /*out_token_time_end*/) { - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - std::vector empty_images; - return decode_multimodal(tokens, empty_images, &audio_features, num_frames, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -uint32_t Gemma4MmModel::decode_with_media( - const std::vector& tokens, - const std::vector& image_paths, - const std::vector& audio_features, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float min_p, float repetition_penalty) { - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - return decode_multimodal(tokens, image_paths, &audio_features, num_frames, - temperature, top_p, top_k, profile_file, out_entropy, - min_p, repetition_penalty); -} - -std::vector Gemma4MmModel::get_image_embeddings(const std::string& image_path) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - auto preprocessed = vision_encoder_.preprocess_image(image_path); - size_t vision_output = vision_encoder_.forward_vision(gb, preprocessed, backend); - size_t projected = vision_encoder_.build_vision_projector(gb, vision_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -std::vector Gemma4MmModel::get_audio_embeddings(const std::vector& audio_features) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t num_frames = audio_features.size() / config_.audio_input_feat_size; - size_t audio_output = audio_encoder_.forward_audio(gb, audio_features, num_frames, backend); - size_t projected = audio_encoder_.build_audio_projector(gb, audio_output, backend); - - gb->execute(); - - const auto& buf = gb->get_output_buffer(projected); - size_t total = buf.total_size; - std::vector embedding(total); - const __fp16* fp16_data = buf.data_as<__fp16>(); - for (size_t i = 0; i < total; i++) - embedding[i] = static_cast(fp16_data[i]); - return embedding; -} - -size_t Gemma4MmModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Gemma4MmModel"); -} - -size_t Gemma4MmModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Gemma4MmModel"); -} - -} -} diff --git a/cactus/models/gemma4/model_gemma4_vision.cpp b/cactus/models/gemma4/model_gemma4_vision.cpp deleted file mode 100644 index 33043db1a..000000000 --- a/cactus/models/gemma4/model_gemma4_vision.cpp +++ /dev/null @@ -1,565 +0,0 @@ -#include "model_gemma4.h" -#include "../../graph/graph.h" -#include -#include -#include -#include -#include -#include - -extern "C" { - #include "../../../libs/stb/stb_image.h" - #include "../../../libs/stb/stb_image_resize2.h" -} - -namespace cactus { -namespace engine { - -static std::pair, std::vector> compute_2d_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - size_t half_dim = head_dim / 2; - size_t freq_per_dim = half_dim / 2; - - std::vector cos_table(max_patches * head_dim, 1.0f); - std::vector sin_table(max_patches * head_dim, 0.0f); - - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t patch_idx = py * img.patch_width + px; - - for (size_t i = 0; i < freq_per_dim; i++) { - float inv_freq = 1.0f / std::pow(theta, 2.0f * i / static_cast(half_dim)); - - float x_angle = static_cast(px) * inv_freq; - cos_table[patch_idx * head_dim + i] = std::cos(x_angle); - cos_table[patch_idx * head_dim + freq_per_dim + i] = std::cos(x_angle); - sin_table[patch_idx * head_dim + i] = std::sin(x_angle); - sin_table[patch_idx * head_dim + freq_per_dim + i] = std::sin(x_angle); - - float y_angle = static_cast(py) * inv_freq; - cos_table[patch_idx * head_dim + half_dim + i] = std::cos(y_angle); - cos_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::cos(y_angle); - sin_table[patch_idx * head_dim + half_dim + i] = std::sin(y_angle); - sin_table[patch_idx * head_dim + half_dim + freq_per_dim + i] = std::sin(y_angle); - } - } - } - - return {cos_table, sin_table}; -} - -static std::pair, std::vector<__fp16>> compute_npu_rope_tables( - const Gemma4VisionModel::PreprocessedImage& img, size_t max_patches, size_t head_dim, float theta) { - - auto [cos_f32, sin_f32] = compute_2d_rope_tables(img, max_patches, head_dim, theta); - - std::vector<__fp16> cos_full(max_patches * head_dim); - std::vector<__fp16> sin_signed(max_patches * head_dim); - - for (size_t p = 0; p < max_patches; p++) { - size_t base = p * head_dim; - for (size_t d = 0; d < head_dim; d++) { - cos_full[base + d] = static_cast<__fp16>(cos_f32[base + d]); - sin_signed[base + d] = static_cast<__fp16>(sin_f32[base + d]); - } - } - - return {cos_full, sin_signed}; -} - -Gemma4VisionModel::Gemma4VisionModel() : Model() {} -Gemma4VisionModel::Gemma4VisionModel(const Config& config) : Model(config) {} - -void Gemma4VisionModel::load_weights_to_graph(CactusGraph* gb) { - auto resolve = [&](const std::string& name) -> std::string { - return model_folder_path_ + "/" + name; - }; - - if (!disable_npu_ && npu::is_npu_available()) { - std::string npu_path = model_folder_path_ + "/vision_encoder.mlpackage"; - if (std::filesystem::exists(npu_path)) { - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.empty()) { - input_shape = {static_cast(config_.vision_default_output_length * - config_.vision_pooling_kernel_size * config_.vision_pooling_kernel_size), - static_cast(config_.vision_embed_dim)}; - } - npu_encoder_->preallocate(input_shape, "hidden_states", "output"); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - CACTUS_LOG_WARN("npu", "[gemma4-vision] found vision_encoder.mlpackage but failed to enable NPU vision encoder; using CPU"); - } - } else { - CACTUS_LOG_WARN("npu", "[gemma4-vision] vision_encoder.mlpackage not found; using CPU vision encoder"); - } - } else if (!disable_npu_) { - CACTUS_LOG_WARN("npu", "[gemma4-vision] NPU backend unavailable on this device; using CPU vision encoder"); - } - - vision_weights_.patch_input_proj = gb->mmap_weights(resolve("vision_patch_embedder_input_proj.weights")); - vision_weights_.position_table = gb->mmap_weights(resolve("vision_patch_embedder_position_embedding_table.weights")); - - if (!use_npu_encoder_) { - vision_weights_.layers.resize(config_.vision_num_layers); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) { - auto& layer = vision_weights_.layers[i]; - std::string prefix = resolve("vision_encoder_layers_" + std::to_string(i) + "_"); - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q_proj.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k_proj.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v_proj.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_o_proj.weights"); - layer.attn_q_norm = gb->mmap_weights(prefix + "self_attn_q_norm.weights"); - layer.attn_k_norm = gb->mmap_weights(prefix + "self_attn_k_norm.weights"); - layer.input_layernorm = gb->mmap_weights(prefix + "input_layernorm.weights"); - layer.post_attention_layernorm = gb->mmap_weights(prefix + "post_attention_layernorm.weights"); - layer.pre_feedforward_layernorm = gb->mmap_weights(prefix + "pre_feedforward_layernorm.weights"); - layer.post_feedforward_layernorm = gb->mmap_weights(prefix + "post_feedforward_layernorm.weights"); - layer.mlp_gate_proj = gb->mmap_weights(prefix + "mlp_gate_proj.weights"); - layer.mlp_up_proj = gb->mmap_weights(prefix + "mlp_up_proj.weights"); - layer.mlp_down_proj = gb->mmap_weights(prefix + "mlp_down_proj.weights"); - layer.layer_scalar = std::filesystem::exists(prefix + "layer_scalar.weights") ? gb->mmap_weights(prefix + "layer_scalar.weights") : 0; - } - } - - vision_weights_.embed_vision_proj = gb->mmap_weights(resolve("embed_vision_proj.weights")); - - size_t vision_head_dim = config_.vision_head_dim; - vision_v_norm_ones_.assign(vision_head_dim, static_cast<__fp16>(1.0f)); - vision_v_norm_ones_node_ = gb->input({vision_head_dim}, Precision::FP16); - gb->set_external_input(vision_v_norm_ones_node_, vision_v_norm_ones_.data(), Precision::FP16); - - vision_weights_.post_proj_norm = gb->mmap_weights(resolve("embed_vision_post_proj_norm.weights")); - - output_weight_node_id_ = 0; -} - -Gemma4VisionModel::PreprocessedImage Gemma4VisionModel::preprocess_image(const std::string& image_path) { - int w, h, channels; - unsigned char* data = stbi_load(image_path.c_str(), &w, &h, &channels, 3); - if (!data) - throw std::runtime_error("Failed to load image: " + image_path); - - uint32_t patch_size = config_.vision_patch_size; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - uint32_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - uint32_t side_mult = pooling_k * patch_size; - - float total_px = static_cast(h) * static_cast(w); - float target_px = static_cast(max_patches) * static_cast(patch_size * patch_size); - float factor = std::sqrt(target_px / total_px); - - int target_h = static_cast(std::floor(factor * h / side_mult)) * side_mult; - int target_w = static_cast(std::floor(factor * w / side_mult)) * side_mult; - - if (target_h == 0) target_h = side_mult; - if (target_w == 0) target_w = side_mult; - - std::vector resized_data; - unsigned char* src = data; - if (target_h != h || target_w != w) { - resized_data.resize(target_h * target_w * 3); - stbir_resize_uint8_linear(data, w, h, 0, resized_data.data(), target_w, target_h, 0, - static_cast(3)); - src = resized_data.data(); - } - - size_t ph = target_h / patch_size; - size_t pw = target_w / patch_size; - size_t num_patches = ph * pw; - - std::vector pixels(3 * target_h * target_w); - for (int y = 0; y < target_h; y++) { - for (int x = 0; x < target_w; x++) { - size_t src_idx = (y * target_w + x) * 3; - for (int c = 0; c < 3; c++) { - pixels[c * target_h * target_w + y * target_w + x] = - static_cast(src[src_idx + c]) * config_.rescale_factor; - } - } - } - - stbi_image_free(data); - - return PreprocessedImage{ - std::move(pixels), - static_cast(target_h), - static_cast(target_w), - ph, pw, num_patches - }; -} - -size_t Gemma4VisionModel::build_vision_patch_embedding(CactusGraph* gb, const PreprocessedImage& img, - ComputeBackend backend) { - uint32_t patch_size = config_.vision_patch_size; - size_t patch_dim = 3 * patch_size * patch_size; - size_t num_patches = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - size_t pos_embed_size = config_.vision_position_embedding_size; - - size_t pixel_input = gb->input({3, img.height, img.width}, Precision::FP32); - gb->set_input(pixel_input, img.pixel_values.data(), Precision::FP32); - - size_t pixel_fp16 = gb->precision_cast(pixel_input, Precision::FP16); - size_t reshaped = gb->reshape(pixel_fp16, {3, img.patch_height, patch_size, img.patch_width, patch_size}); - size_t permuted = gb->transposeN(reshaped, {1, 3, 2, 4, 0}); - size_t patches = gb->reshape(permuted, {num_patches, patch_dim}); - - float mean = config_.image_mean; - float norm_std = config_.image_std; - patches = gb->scalar_multiply(gb->scalar_add(patches, -mean), 1.0f / norm_std); - - size_t projected = gb->matmul(patches, vision_weights_.patch_input_proj, true, backend); - - size_t pos_table = gb->reshape(vision_weights_.position_table, {2, pos_embed_size, hidden_size}); - - std::vector x_positions(num_patches); - std::vector y_positions(num_patches); - for (size_t py = 0; py < img.patch_height; py++) { - for (size_t px = 0; px < img.patch_width; px++) { - size_t idx = py * img.patch_width + px; - x_positions[idx] = static_cast(px); - y_positions[idx] = static_cast(py); - } - } - - size_t x_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(x_pos_input, x_positions.data(), Precision::FP32); - size_t y_pos_input = gb->input({num_patches}, Precision::FP32); - gb->set_input(y_pos_input, y_positions.data(), Precision::FP32); - - size_t x_table = gb->slice(pos_table, 0, 0, 1); - x_table = gb->reshape(x_table, {pos_embed_size, hidden_size}); - size_t y_table = gb->slice(pos_table, 0, 1, 1); - y_table = gb->reshape(y_table, {pos_embed_size, hidden_size}); - - size_t x_embed = gb->embedding(x_table, x_pos_input); - size_t y_embed = gb->embedding(y_table, y_pos_input); - size_t pos_embed = gb->add(x_embed, y_embed); - - size_t pos_embed_fp16 = gb->precision_cast(pos_embed, Precision::FP16); - - return gb->add(projected, pos_embed_fp16); -} - -std::pair Gemma4VisionModel::build_2d_rope_nodes( - CactusGraph* gb, const PreprocessedImage& img, size_t max_patches) { - - size_t head_dim = config_.vision_head_dim; - auto rope_tables = compute_2d_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - auto& cos_table = rope_tables.first, sin_table = rope_tables.second; - - size_t cos_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(cos_input, cos_table.data(), Precision::FP32); - size_t cos_node = gb->precision_cast(cos_input, Precision::FP16); - cos_node = gb->reshape(cos_node, {1, max_patches, 1, head_dim}); - - size_t sin_input = gb->input({max_patches, head_dim}, Precision::FP32); - gb->set_input(sin_input, sin_table.data(), Precision::FP32); - size_t sin_node = gb->precision_cast(sin_input, Precision::FP16); - sin_node = gb->reshape(sin_node, {1, max_patches, 1, head_dim}); - - return {cos_node, sin_node}; -} - -size_t Gemma4VisionModel::build_padding_mask(CactusGraph* gb, size_t num_real, size_t max_patches) { - if (num_real >= max_patches) - return 0; - - std::vector mask(max_patches * max_patches, 0.0f); - for (size_t q = 0; q < num_real; q++) - for (size_t k = 0; k < num_real; k++) - mask[q * max_patches + k] = 1.0f; - size_t mask_input = gb->input({max_patches, max_patches}, Precision::FP32); - gb->set_input(mask_input, mask.data(), Precision::FP32); - size_t mask_node = gb->precision_cast(mask_input, Precision::FP16); - return gb->reshape(mask_node, {1, max_patches, max_patches}); -} - -size_t Gemma4VisionModel::build_vision_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - size_t head_dim = config_.vision_head_dim; - size_t num_heads = config_.vision_attention_heads; - size_t kv_heads = config_.vision_kv_heads; - - const auto& input_buf = gb->get_output_buffer(input); - size_t seq_len = input_buf.shape[0]; - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - auto v = gb->rms_norm(gb->reshape(v_proj, {seq_len * kv_heads, head_dim}), - vision_v_norm_ones_node_, config_.layer_norm_eps); - v = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - size_t half_dim = head_dim / 2; - - auto apply_2d_rope = [&](size_t tensor) -> size_t { - auto first_half = gb->slice(tensor, 3, 0, half_dim); - auto second_half = gb->slice(tensor, 3, half_dim, half_dim); - - auto rot_first_a = gb->slice(first_half, 3, 0, half_dim / 2); - auto rot_first_b = gb->slice(first_half, 3, half_dim / 2, half_dim / 2); - auto neg_b = gb->scalar_multiply(rot_first_b, -1.0f); - - size_t cos_x = gb->slice(cos_node, 3, 0, half_dim); - size_t sin_x = gb->slice(sin_node, 3, 0, half_dim); - - auto first_rotated = gb->add( - gb->multiply(first_half, cos_x), - gb->multiply(gb->concat(neg_b, rot_first_a, 3), sin_x)); - - auto rot_second_a = gb->slice(second_half, 3, 0, half_dim / 2); - auto rot_second_b = gb->slice(second_half, 3, half_dim / 2, half_dim / 2); - auto neg_sb = gb->scalar_multiply(rot_second_b, -1.0f); - - size_t cos_y = gb->slice(cos_node, 3, half_dim, half_dim); - size_t sin_y = gb->slice(sin_node, 3, half_dim, half_dim); - - auto second_rotated = gb->add( - gb->multiply(second_half, cos_y), - gb->multiply(gb->concat(neg_sb, rot_second_a, 3), sin_y)); - - return gb->concat(first_rotated, second_rotated, 3); - }; - - auto q_rot = apply_2d_rope(q); - auto k_rot = apply_2d_rope(k); - - size_t attn; - if (attn_mask_node != 0) - attn = gb->attention_masked(q_rot, k_rot, v, attn_mask_node, 1.f, false, backend); - else - attn = gb->attention(q_rot, k_rot, v, 1.f, false, backend); - - return gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), - layer.attn_output_weight, true, backend); -} - -size_t Gemma4VisionModel::build_vision_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - auto gate = gb->gelu(gb->matmul(input, layer.mlp_gate_proj, true, backend)); - auto up = gb->matmul(input, layer.mlp_up_proj, true, backend); - return gb->matmul(gb->multiply(gate, up), layer.mlp_down_proj, true, backend); -} - -size_t Gemma4VisionModel::build_vision_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - size_t cos_node, size_t sin_node, - size_t attn_mask_node, ComputeBackend backend) { - const auto& layer = vision_weights_.layers[layer_idx]; - - auto normed = gb->rms_norm(hidden, layer.input_layernorm, config_.layer_norm_eps); - auto attn_raw = build_vision_attention(gb, normed, layer_idx, cos_node, sin_node, attn_mask_node, backend); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm, config_.layer_norm_eps); - auto residual = gb->add(hidden, attn); - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm, config_.layer_norm_eps); - auto mlp_raw = build_vision_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm, config_.layer_norm_eps); - - auto out = gb->add(residual, mlp); - - if (layer.layer_scalar != 0) - out = gb->multiply(out, layer.layer_scalar); - - return out; -} - -size_t Gemma4VisionModel::build_vision_pooler(CactusGraph* gb, size_t hidden, const PreprocessedImage& img, - ComputeBackend backend) { - size_t k = config_.vision_pooling_kernel_size; - size_t output_length = config_.vision_default_output_length; - size_t num_patches = img.num_patches; - size_t k_squared = k * k; - - if (num_patches == output_length) - return hidden; - - size_t max_x = img.patch_width; - size_t max_y = img.patch_height; - - std::vector pool_weights(num_patches * output_length, 0.0f); - std::vector valid_bins(output_length, false); - for (size_t py = 0; py < max_y; py++) { - for (size_t px = 0; px < max_x; px++) { - size_t patch_idx = py * max_x + px; - size_t kx = px / k; - size_t ky = py / k; - size_t kernel_idx = kx + (max_x / k) * ky; - if (kernel_idx < output_length) { - pool_weights[kernel_idx * num_patches + patch_idx] = 1.0f / static_cast(k_squared); - valid_bins[kernel_idx] = true; - } - } - } - - size_t weights_node = gb->input({output_length, num_patches}, Precision::FP32); - gb->set_input(weights_node, pool_weights.data(), Precision::FP32); - size_t weights_fp16 = gb->precision_cast(weights_node, Precision::FP16); - - size_t pooled = gb->matmul(weights_fp16, hidden, false, backend); - - size_t valid_count = 0; - for (bool v : valid_bins) if (v) valid_count++; - - if (valid_count == output_length) - return pooled; - - std::vector valid_slices; - for (size_t i = 0; i < output_length; i++) { - if (valid_bins[i]) { - auto slice = gb->slice(pooled, 0, i, 1); - valid_slices.push_back(slice); - } - } - - size_t stripped = valid_slices[0]; - for (size_t i = 1; i < valid_slices.size(); i++) - stripped = gb->concat(stripped, valid_slices[i], 0); - - return stripped; -} - -size_t Gemma4VisionModel::forward_vision(CactusGraph* gb, const PreprocessedImage& img, ComputeBackend backend) { - size_t hidden = build_vision_patch_embedding(gb, img, backend); - - size_t num_real = img.num_patches; - size_t hidden_size = config_.vision_embed_dim; - uint32_t pooling_k = config_.vision_pooling_kernel_size; - size_t max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - bool can_use_npu_path = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); - if (can_use_npu_path) { - std::vector npu_input_shape = npu_encoder_->get_input_shape(); - if (npu_input_shape.size() >= 2) { - size_t a = static_cast(npu_input_shape[0]); - size_t b = static_cast(npu_input_shape[1]); - if (b == hidden_size) { - max_patches = a; - } else if (a == hidden_size) { - max_patches = b; - } else if (a > 0) { - max_patches = a; - } - } - } - if (num_real > max_patches) { - if (can_use_npu_path) { - CACTUS_LOG_WARN("npu", "[gemma4-vision] image has more patches than NPU encoder supports; falling back to CPU vision encoder"); - can_use_npu_path = false; - max_patches = config_.vision_default_output_length * pooling_k * pooling_k; - } - } - if (!can_use_npu_path) { - max_patches = num_real; - } - size_t num_padding = max_patches - num_real; - - if (num_padding > 0) { - std::vector<__fp16> zeros(num_padding * hidden_size, static_cast<__fp16>(0.0f)); - size_t pad_node = gb->input({num_padding, hidden_size}, Precision::FP16); - gb->set_input(pad_node, zeros.data(), Precision::FP16); - hidden = gb->concat(hidden, pad_node, 0); - } - - auto [cos_node, sin_node] = build_2d_rope_nodes(gb, img, max_patches); - size_t attn_mask_node = build_padding_mask(gb, num_real, max_patches); - - if (can_use_npu_path) { - gb->execute(); - - const auto& h_buf = gb->get_output_buffer(hidden); - if (h_buf.precision != Precision::FP16) - throw std::runtime_error("[gemma4-vision] expected FP16 hidden output for NPU path"); - std::vector<__fp16> hidden_fp16(max_patches * hidden_size); - memcpy(hidden_fp16.data(), h_buf.data_as<__fp16>(), max_patches * hidden_size * sizeof(__fp16)); - - size_t head_dim = config_.vision_head_dim; - auto [cos_fp16, sin_fp16] = compute_npu_rope_tables(img, max_patches, head_dim, config_.vision_rope_theta); - - std::vector<__fp16> additive_mask(max_patches * max_patches, static_cast<__fp16>(0.0f)); - if (num_padding > 0) { - for (size_t q = 0; q < max_patches; q++) - for (size_t k = 0; k < max_patches; k++) - if (q >= num_real || k >= num_real) - additive_mask[q * max_patches + k] = static_cast<__fp16>(-65504.0f); - } - - std::vector inputs = { - {"hidden_states", hidden_fp16.data(), {static_cast(max_patches), static_cast(hidden_size)}}, - {"cos_full", cos_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"sin_signed", sin_fp16.data(), {static_cast(max_patches), 1, static_cast(head_dim)}}, - {"attention_mask", additive_mask.data(), {1, static_cast(max_patches), static_cast(max_patches)}}, - }; - - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t out_elements = 1; - for (int d : out_shape) out_elements *= d; - size_t required = max_patches * hidden_size; - if (out_elements < required) - out_elements = required; - - std::vector<__fp16> npu_output(out_elements); - size_t written = npu_encoder_->encode_multimodal_input(inputs, npu_output.data(), "output"); - - if (written >= required) { - hidden = gb->input({max_patches, hidden_size}, Precision::FP16); - gb->set_input(hidden, npu_output.data(), Precision::FP16); - } else { - CACTUS_LOG_WARN("npu", "[gemma4-vision] encode_multimodal_input returned insufficient output, falling back to CPU"); - for (uint32_t i = 0; i < config_.vision_num_layers; i++) - hidden = build_vision_transformer_block(gb, hidden, i, cos_node, sin_node, attn_mask_node, backend); - } - } else { - for (uint32_t i = 0; i < config_.vision_num_layers; i++) { - auto [cn, sn] = build_2d_rope_nodes(gb, img, num_real); - hidden = build_vision_transformer_block(gb, hidden, i, cn, sn, 0, backend); - gb->execute(); - - const auto& out_buf = gb->get_output_buffer(hidden); - std::vector tmp(out_buf.byte_size); - std::memcpy(tmp.data(), out_buf.get_data(), out_buf.byte_size); - auto shape = out_buf.shape; - auto prec = out_buf.precision; - - const auto& lw = vision_weights_.layers[i]; - for (size_t wn : {lw.attn_q_weight, lw.attn_k_weight, lw.attn_v_weight, - lw.attn_output_weight, lw.mlp_gate_proj, lw.mlp_up_proj, lw.mlp_down_proj}) - gb->release_weight_pages(wn); - - gb->soft_reset_keep_pool(); - hidden = gb->input(shape, prec); - gb->set_input(hidden, tmp.data(), prec); - } - gb->release_weight_pages(vision_weights_.patch_input_proj); - gb->release_weight_pages(vision_weights_.position_table); - } - - if (num_padding > 0) - hidden = gb->slice(hidden, 0, 0, num_real); - - return build_vision_pooler(gb, hidden, img, backend); -} - -size_t Gemma4VisionModel::build_vision_projector(CactusGraph* gb, size_t vision_features, ComputeBackend backend) { - size_t projected = gb->matmul(vision_features, vision_weights_.embed_vision_proj, true, backend); - return gb->rms_norm(projected, vision_weights_.post_proj_norm, config_.layer_norm_eps); -} - -} -} diff --git a/cactus/models/model.h b/cactus/models/model.h deleted file mode 100644 index 2cb17a0e4..000000000 --- a/cactus/models/model.h +++ /dev/null @@ -1,1775 +0,0 @@ -#pragma once - -#include "../engine/engine.h" -#include "../npu/npu.h" - -namespace cactus { -namespace engine { - - - -class QwenModel : public Model { -public: - QwenModel(); - explicit QwenModel(const Config& config); - ~QwenModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class Qwen3p5Model : public Model { -public: - Qwen3p5Model(); - explicit Qwen3p5Model(const Config& config); - ~Qwen3p5Model() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - -private: - size_t build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset); - - struct WeightNodeIDs { - size_t output_weight = 0; - size_t output_norm_weight = 0; - - enum class LayerType : uint8_t { ATTENTION, DELTANET }; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t input_layernorm_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - size_t deltanet_qkv_weight = 0; - size_t deltanet_gate_weight = 0; - size_t deltanet_beta_weight = 0; - size_t deltanet_gate_bias = 0; - size_t deltanet_beta_bias = 0; - size_t deltanet_z_weight = 0; - size_t deltanet_conv_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - size_t post_attention_layernorm_weight = 0; - }; - - struct LayerEntry { - LayerType type = LayerType::ATTENTION; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_state_nodes_; - bool last_forward_used_cache_ = false; - size_t deltanet_total_seq_len_ = 0; - size_t deltanet_heads_ = 0; - size_t deltanet_key_dim_ = 0; - size_t deltanet_value_dim_ = 0; - size_t deltanet_state_flat_dim_ = 0; - size_t deltanet_mixed_dim_ = 0; - size_t deltanet_conv_history_len_ = 0; - size_t deltanet_conv_flat_dim_ = 0; - size_t deltanet_cache_row_dim_ = 0; -}; - - - -class GemmaModel : public Model { -public: - GemmaModel(); - explicit GemmaModel(const Config& config); - ~GemmaModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class GemmaModel3n : public Model { -public: - GemmaModel3n(); - explicit GemmaModel3n(const Config& config); - ~GemmaModel3n() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - void load_weights_to_graph(CactusGraph* gb) override; - void post_init() override; - std::vector get_kv_layer_dims() const override; - -private: - size_t forward_split(const std::vector& tokens, bool use_cache); - - size_t build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input, size_t* streams); - void build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t pos_offset, size_t pli, size_t* streams); - size_t build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend); - void set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens); - - size_t build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, ComputeBackend backend) const; - size_t build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const; - size_t build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const; - size_t build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const; - - size_t build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, ComputeBackend backend) const; - void build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, - const size_t* streams, size_t* predictions) const; - void build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, - ComputeBackend backend, const size_t* predictions, size_t* corrected) const; - void build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend, size_t* streams) const; - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t altup_proj_weights[3]; - size_t altup_unembed_proj_weights[3]; - size_t embed_tokens_per_layer; - size_t per_layer_model_proj; - size_t per_layer_proj_norm; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t input_layernorm_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - size_t pre_feedforward_layernorm_weight; - size_t post_feedforward_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - size_t post_attention_layernorm_weight; - - size_t altup_router_norm; - size_t altup_prediction_coefs; - size_t altup_correction_coefs; - size_t altup_correct_output_scale; - size_t altup_modality_router; - size_t laurel_left; - size_t laurel_right; - size_t laurel_norm; - size_t per_layer_gate; - size_t per_layer_proj; - size_t post_per_layer_norm; - }; - - std::vector layers; - } weight_nodes_; - - std::vector kv_share_map_; - std::vector shared_k_nodes_; - std::vector shared_v_nodes_; - - std::vector<__fp16> v_norm_ones_weight_; - size_t v_norm_ones_node_ = 0; -}; - -class Siglip2VisionModel : public Model { - friend class Lfm2VlModel; - -public: - struct VisionEmbeddingResult { - size_t combined_embeddings; - std::vector tile_embeddings; - }; - - Siglip2VisionModel(); - explicit Siglip2VisionModel(const Config& cfg); - ~Siglip2VisionModel() override = default; - virtual size_t forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image); - virtual size_t forward_vision(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - std::vector get_image_embedding(const std::string& image_path); - Siglip2Preprocessor& get_preprocessor() { return preprocessor_; } - const Siglip2Preprocessor& get_preprocessor() const { return preprocessor_; } - -protected: - VisionEmbeddingResult build_vision_embeddings(CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - - size_t build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_vision_attention(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_vision_mlp(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx, - ComputeBackend backend); - void ensure_cpu_vision_weights_loaded(CactusGraph* gb); - - void load_weights_to_graph(CactusGraph* gb) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - -protected: - struct VisionWeightNodeIDs { - size_t patch_embedding_weight; - size_t patch_embedding_bias; - size_t position_embedding; - size_t post_layernorm_weight; - size_t post_layernorm_bias; - - struct VisionLayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_bias; - size_t attn_k_bias; - size_t attn_v_bias; - size_t attn_output_bias; - size_t layer_norm1_weight; - size_t layer_norm1_bias; - size_t layer_norm2_weight; - size_t layer_norm2_bias; - size_t mlp_fc1_weight; - size_t mlp_fc1_bias; - size_t mlp_fc2_weight; - size_t mlp_fc2_bias; - }; - - std::vector vision_layers; - } vision_weight_nodes_; - - Siglip2Preprocessor preprocessor_; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool cpu_vision_weights_loaded_ = false; -}; - - -class LFM2Model : public Model { - friend class Lfm2VlModel; - -public: - LFM2Model(); - explicit LFM2Model(const Config& config); - ~LFM2Model() override = default; - - bool is_cache_empty() const; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - -protected: - using Model::forward; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache); - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); - size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_output_weight; - size_t attn_q_norm_weight; - size_t attn_k_norm_weight; - - size_t conv_depthwise_weight; - size_t conv_in_proj_weight; - size_t conv_out_proj_weight; - - size_t input_layernorm_weight; - size_t post_attention_layernorm_weight; - size_t ffn_gate_weight; - size_t ffn_up_weight; - size_t ffn_down_weight; - }; - - enum class LayerType : uint8_t { ATTENTION, CONV }; - - struct LayerEntry { - LayerType type; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_bx_nodes_; - bool last_forward_used_cache_ = false; -}; - - -class LFM2MoEModel : public Model { -public: - LFM2MoEModel(); - explicit LFM2MoEModel(const Config& config); - ~LFM2MoEModel() override = default; - - bool is_cache_empty() const; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - bool init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt = "", bool do_warmup = true) override; - -protected: - using Model::forward; - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache); - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - size_t forward(CactusGraph* gb, const std::vector& tokens, ComputeBackend backend, bool use_cache = false); - size_t forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, ComputeBackend backend, bool use_cache = false); - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - void reset_cache() override; - void load_weights_to_graph(CactusGraph* gb) override; - -private: - bool is_dense_layer(uint32_t layer_idx) const; - - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - struct ExpertWeights { - size_t w1_weight = 0; - size_t w3_weight = 0; - size_t w2_weight = 0; - }; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - - size_t conv_depthwise_weight = 0; - size_t conv_in_proj_weight = 0; - size_t conv_out_proj_weight = 0; - - size_t input_layernorm_weight = 0; - size_t post_attention_layernorm_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - - size_t moe_router_weight = 0; - size_t moe_expert_bias = 0; - std::vector moe_experts; - }; - - enum class LayerType : uint8_t { ATTENTION, CONV }; - - struct LayerEntry { - LayerType type; - LayerWeights weights; - }; - - std::vector layers; - } weight_nodes_; - - ConvCache conv_cache_; - std::vector conv_cache_bx_nodes_; - bool last_forward_used_cache_ = false; -}; - - -class NomicModel : public Model { -public: - NomicModel(); - explicit NomicModel(const Config& config); - ~NomicModel() override = default; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - size_t build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - struct WeightNodeIDs { - size_t embedding_layernorm_weight; - size_t embedding_layernorm_bias; - - struct LayerWeights { - size_t attn_q_weight; - size_t attn_k_weight; - size_t attn_v_weight; - size_t attn_q_bias; - size_t attn_k_bias; - size_t attn_v_bias; - size_t attn_output_weight; - size_t attn_output_bias; - size_t ffn_up_weight; - size_t ffn_up_bias; - size_t ffn_norm_1_weight; - size_t ffn_norm_1_bias; - size_t ffn_down_weight; - size_t ffn_down_bias; - size_t ffn_norm_2_weight; - size_t ffn_norm_2_bias; - size_t mlp_router_layer_weight; - size_t mlp_experts_bias; - std::vector mlp_experts_mlp1_weight; - std::vector mlp_experts_mlp2_weight; - }; - - std::vector layers; - } weight_nodes_; -}; - -class WhisperModel : public Model { -public: - WhisperModel(); - explicit WhisperModel(const Config& config); - ~WhisperModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { - throw std::runtime_error("Whisper: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Whisper: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Whisper: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Whisper requires mel+token forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - void run_encoder(const std::vector& audio_features); - void reset_graph_side_cache_nodes(); - - size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); - - void load_weights_to_graph(CactusGraph* gb) override; - - size_t build_encoder_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_conv1d(CactusGraph* gb, size_t input); - - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - std::vector get_audio_embeddings(const std::vector& audio_features) override; - - void reset_cache() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - size_t output_norm_weight; - - size_t decoder_norm_weight; - size_t decoder_norm_bias; - size_t decoder_position_embeddings_weight; - - size_t encoder_position_embeddings; - size_t encoder_conv1_weight; - size_t encoder_conv1_bias; - size_t encoder_conv2_weight; - size_t encoder_conv2_bias; - size_t encoder_conv3_weight; - size_t encoder_conv3_bias; - size_t encoder_norm_weight; - size_t encoder_norm_bias; - - size_t encoder_output; - - struct LayerWeights { - size_t decoder_output_norm_bias; - size_t decoder_output_norm_weight; - size_t decoder_position_embeddings_weight; - size_t decoder_token_embeddings_weight; - - size_t decoder_encoder_attn_q_weight; - size_t decoder_encoder_attn_k_weight; - size_t decoder_encoder_attn_v_weight; - size_t decoder_encoder_attn_q_bias; - size_t decoder_encoder_attn_v_bias; - size_t decoder_encoder_attn_output_weight; - size_t decoder_encoder_attn_output_bias; - - size_t decoder_post_encoder_layernorm_weight; - size_t decoder_post_encoder_layernorm_bias; - - size_t decoder_ffn1_weight; - size_t decoder_ffn1_bias; - size_t decoder_ffn2_weight; - size_t decoder_ffn2_bias; - - size_t decoder_post_ffn_layernorm_weight; - size_t decoder_post_ffn_layernorm_bias; - - size_t decoder_self_attn_q_weight; - size_t decoder_self_attn_k_weight; - size_t decoder_self_attn_v_weight; - size_t decoder_self_attn_q_bias; - size_t decoder_self_attn_v_bias; - size_t decoder_self_attn_output_weight; - size_t decoder_self_attn_output_bias; - - size_t decoder_post_attn_layernorm_weight; - size_t decoder_post_attn_layernorm_bias; - - //Encoder layers - size_t encoder_ffn1_weight; - size_t encoder_ffn1_bias; - size_t encoder_ffn2_weight; - size_t encoder_ffn2_bias; - - size_t encoder_post_ffn_layernorm_weight; - size_t encoder_post_ffn_layernorm_bias; - - size_t encoder_self_attn_q_weight; - size_t encoder_self_attn_k_weight; - size_t encoder_self_attn_v_weight; - size_t encoder_self_attn_q_bias; - size_t encoder_self_attn_v_bias; - size_t encoder_self_attn_output_weight; - size_t encoder_self_attn_output_bias; - - size_t encoder_post_attn_layernorm_weight; - size_t encoder_post_attn_layernorm_bias; - }; - - std::vector layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_new_tokens_; - size_t last_conv1_node_ = 0; - size_t last_conv2_node_ = 0; - size_t last_encoder_post_norm_node_ = 0; - size_t last_enc_plus_pos_node_ = 0; - size_t encoder_transformer_block_0 = 0; - size_t encoder_pre_gelu = 0; - size_t encoder_post_gelu = 0; - - std::vector encoder_block_out_nodes_; - - size_t encoder_output_persistent_ = 0; - - std::vector suppress_tokens_ = { - 1, - 2, - 7, - 8, - 9, - 10, - 14, - 25, - 26, - 27, - 28, - 29, - 31, - 58, - 59, - 60, - 61, - 62, - 63, - 90, - 91, - 92, - 93, - 359, - 503, - 522, - 542, - 873, - 893, - 902, - 918, - 922, - 931, - 1350, - 1853, - 1982, - 2460, - 2627, - 3246, - 3253, - 3268, - 3536, - 3846, - 3961, - 4183, - 4667, - 6585, - 6647, - 7273, - 9061, - 9383, - 10428, - 10929, - 11938, - 12033, - 12331, - 12562, - 13793, - 14157, - 14635, - 15265, - 15618, - 16553, - 16604, - 18362, - 18956, - 20075, - 21675, - 22520, - 26130, - 26161, - 26435, - 28279, - 29464, - 31650, - 32302, - 32470, - 36865, - 42863, - 47425, - 49870, - 50254, - 50258, - 50358, - 50359, - 50360, - 50361, - 50362 - }; - - std::vector begin_suppress_tokens_ = { - 220, - 50257 - }; - - bool first_decode_step_ = true; - - std::unordered_map suppress_bias_; - std::unordered_map suppress_bias_first_step_; - - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - - uint32_t enc_layers_ = 0; - uint32_t dec_layers_ = 0; -}; - - -class MoonshineModel : public Model { -public: - MoonshineModel(); - explicit MoonshineModel(const Config& config); - ~MoonshineModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t,ComputeBackend, bool, size_t) override { - throw std::runtime_error("Moonshine: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Moonshine: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Moonshine: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Moonshine requires mel+token forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - - void reset_graph_side_cache_nodes(); - - size_t build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only); - - void load_weights_to_graph(CactusGraph* gb) override; - - size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_encoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend); - - size_t build_decoder_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const; - - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - - size_t build_conv1d(CactusGraph* gb, size_t input); - size_t build_audio_preprocessor(CactusGraph* gb, size_t input); - - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - - std::vector get_audio_embeddings(const std::vector& audio_features) override; - - void reset_cache() override; - -private: - struct WeightNodeIDs { - size_t output_weight; - - size_t decoder_norm_weight; - - size_t encoder_conv1_weight; - size_t encoder_conv2_weight; - size_t encoder_conv2_bias; - size_t encoder_conv3_weight; - size_t encoder_conv3_bias; - size_t encoder_norm_weight; - size_t encoder_norm_bias; - size_t encoder_layer_norm_weight; - - - struct LayerWeights { - //Decoder layers - - size_t decoder_encoder_attn_q_weight; - size_t decoder_encoder_attn_k_weight; - size_t decoder_encoder_attn_v_weight; - size_t decoder_encoder_attn_output_weight; - - size_t decoder_post_encoder_layernorm_weight; - - size_t decoder_ffn1_weight; - size_t decoder_ffn1_bias; - size_t decoder_ffn_gate_weight; - size_t decoder_ffn_gate_bias; - size_t decoder_ffn_up_weight; - size_t decoder_ffn_up_bias; - size_t decoder_ffn2_weight; - size_t decoder_ffn2_bias; - - size_t decoder_post_ffn_layernorm_weight; - - size_t decoder_self_attn_q_weight; - size_t decoder_self_attn_k_weight; - size_t decoder_self_attn_v_weight; - size_t decoder_self_attn_output_weight; - - size_t decoder_post_attn_layernorm_weight; - - //Encoder layers - size_t encoder_ffn1_weight; - size_t encoder_ffn1_bias; - size_t encoder_ffn_gate_weight; - size_t encoder_ffn_gate_bias; - size_t encoder_ffn_up_weight; - size_t encoder_ffn_up_bias; - size_t encoder_ffn2_weight; - size_t encoder_ffn2_bias; - - size_t encoder_post_ffn_layernorm_weight; - - size_t encoder_self_attn_q_weight; - size_t encoder_self_attn_k_weight; - size_t encoder_self_attn_v_weight; - size_t encoder_self_attn_output_weight; - - size_t encoder_post_attn_layernorm_weight; - }; - - std::vector encoder_layers; - std::vector decoder_layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_new_tokens_; - std::vector encoder_output_host_; - std::vector encoder_output_shape_; - size_t last_conv1_node_ = 0; - size_t last_conv2_node_ = 0; - size_t last_encoder_post_norm_node_ = 0; - size_t last_enc_plus_pos_node_ = 0; - size_t encoder_transformer_block_0 = 0; - size_t encoder_pre_gelu = 0; - size_t encoder_post_gelu = 0; - - std::vector encoder_block_out_nodes_; - std::vector encoder_output_bytes_; - Precision encoder_output_precision_ = Precision::FP32; - - bool first_decode_step_ = true; - - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; - - Precision encoder_kv_precision_ = Precision::FP32; - bool encoder_kv_ready_ = false; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - -}; - -class NeedleModel : public Model { -public: - NeedleModel(); - explicit NeedleModel(const Config& config); - ~NeedleModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Needle: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Needle: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Needle: build_transformer_block unused"); - } - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - uint32_t decode(const std::vector& tokens, float temperature = -1.0f, float top_p = -1.0f, - size_t top_k = 0, const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f) override; - void load_weights_to_graph(CactusGraph* gb) override; - void reset_cache() override; - - void run_encoder(const std::vector& encoder_tokens); - void reset_graph_side_cache_nodes(); - size_t run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only); - - size_t build_encoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_self_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_cross_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_encoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - size_t build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0); - -private: - struct WeightNodeIDs { - size_t output_weight = 0; - size_t encoder_norm_weight = 0; - size_t decoder_norm_weight = 0; - - struct EncoderLayerWeights { - size_t attn_gate_weight = 0; - size_t attn_q_weight = 0; - size_t attn_k_weight = 0; - size_t attn_v_weight = 0; - size_t attn_output_weight = 0; - size_t input_norm_weight = 0; - size_t attn_q_norm_weight = 0; - size_t attn_k_norm_weight = 0; - }; - - struct DecoderLayerWeights { - size_t self_attn_gate_weight = 0; - size_t cross_attn_gate_weight = 0; - size_t self_attn_q_weight = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_q_norm_weight = 0; - size_t self_attn_k_norm_weight = 0; - - size_t encoder_attn_q_weight = 0; - size_t encoder_attn_k_weight = 0; - size_t encoder_attn_v_weight = 0; - size_t encoder_attn_output_weight = 0; - size_t encoder_attn_q_norm_weight = 0; - size_t encoder_attn_k_norm_weight = 0; - - size_t input_norm_weight = 0; - size_t post_attn_norm_weight = 0; - }; - - std::vector encoder_layers; - std::vector decoder_layers; - } weight_nodes_; - - bool encoder_ready_ = false; - size_t last_encoder_post_norm_node_ = 0; - std::vector encoder_k_persistent_; - std::vector encoder_v_persistent_; -}; - -class ParakeetModel : public Model { -public: - ParakeetModel(); - explicit ParakeetModel(const Config& config); - ~ParakeetModel() override = default; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Parakeet: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("Parakeet: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("Parakeet: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("Parakeet requires audio feature forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void reset_cache() override; - -private: - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - size_t build_ctc_logits(CactusGraph* gb, size_t hidden_states); - size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); - size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); - size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); - size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); - size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - std::vector greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const; - - struct WeightNodeIDs { - size_t ctc_head_weight = 0; - size_t ctc_head_bias = 0; - - size_t subsampling_conv0_weight = 0; - size_t subsampling_conv0_bias = 0; - size_t subsampling_depthwise1_weight = 0; - size_t subsampling_depthwise1_bias = 0; - size_t subsampling_pointwise1_weight = 0; - size_t subsampling_pointwise1_bias = 0; - size_t subsampling_depthwise2_weight = 0; - size_t subsampling_depthwise2_bias = 0; - size_t subsampling_pointwise2_weight = 0; - size_t subsampling_pointwise2_bias = 0; - size_t subsampling_linear_weight = 0; - size_t subsampling_linear_bias = 0; - - struct LayerWeights { - size_t ff1_linear1_weight = 0; - size_t ff1_linear1_bias = 0; - size_t ff1_linear2_weight = 0; - size_t ff1_linear2_bias = 0; - - size_t ff2_linear1_weight = 0; - size_t ff2_linear1_bias = 0; - size_t ff2_linear2_weight = 0; - size_t ff2_linear2_bias = 0; - - size_t self_attn_q_weight = 0; - size_t self_attn_q_bias = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_k_bias = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_v_bias = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_output_bias = 0; - size_t self_attn_relative_k_weight = 0; - size_t self_attn_bias_u = 0; - size_t self_attn_bias_v = 0; - - size_t norm_ff1_weight = 0; - size_t norm_ff1_bias = 0; - size_t norm_self_attn_weight = 0; - size_t norm_self_attn_bias = 0; - size_t norm_conv_weight = 0; - size_t norm_conv_bias = 0; - size_t norm_ff2_weight = 0; - size_t norm_ff2_bias = 0; - size_t norm_out_weight = 0; - size_t norm_out_bias = 0; - - size_t conv_pointwise1_weight = 0; - size_t conv_pointwise1_bias = 0; - size_t conv_depthwise_weight = 0; - size_t conv_depthwise_bias = 0; - size_t conv_pointwise2_weight = 0; - size_t conv_pointwise2_bias = 0; - size_t conv_batchnorm_weight = 0; - size_t conv_batchnorm_bias = 0; - size_t conv_batchnorm_running_mean = 0; - size_t conv_batchnorm_running_var = 0; - }; - - std::vector layers; - } weight_nodes_; - - bool ctc_tokens_ready_ = false; - size_t ctc_emit_index_ = 0; - std::vector ctc_tokens_; - size_t last_input_token_count_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool has_cpu_encoder_weights_ = false; -}; - -class ParakeetTDTModel : public Model { -public: - ParakeetTDTModel(); - explicit ParakeetTDTModel(const Config& config); - ~ParakeetTDTModel() override = default; - - struct TDTToken { uint32_t id; float time_start; float time_end; }; - - struct ChunkStreamState { - bool initialized = false; - uint32_t last_token = 0; - std::vector> h; - std::vector> c; - }; - - struct ChunkStreamResult { - std::string text; - std::string confirmed_text; - std::string pending_text; - size_t token_count = 0; - size_t confirmed_token_count = 0; - double raw_decoder_tps = 0.0; - double raw_decoder_time_ms = 0.0; - float start_sec = 0.0f; - float confirmed_end_sec = 0.0f; - float resume_end_sec = 0.0f; - float end_sec = 0.0f; - }; - - ChunkStreamResult decode_chunk_stream( - const std::vector& audio_features, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState& state); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("ParakeetTDT: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("ParakeetTDT: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("ParakeetTDT: build_transformer_block unused"); - } - - size_t forward(const std::vector& /*tokens*/, bool /*use_cache*/ = false) override { - throw std::runtime_error("ParakeetTDT requires audio feature forward()."); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - uint32_t decode_with_audio(const std::vector& tokens, const std::vector& audio_features, - float temperature = 0.0f, float top_p = 0.0f, size_t top_k = 0, - const std::string& profile_file = "", float* out_entropy = nullptr, - float min_p = 0.15f, float repetition_penalty = 1.1f, - float* out_token_time_start = nullptr, float* out_token_time_end = nullptr) override; - std::vector get_audio_embeddings(const std::vector& audio_features) override; - void reset_cache() override; - -private: - size_t build_encoder(CactusGraph* gb, const std::vector& audio_features); - size_t build_subsampling(CactusGraph* gb, const std::vector& audio_features); - size_t build_relative_position_embeddings(CactusGraph* gb, size_t seq_len); - size_t build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - size_t build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend); - size_t build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend); - size_t build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, uint32_t layer_idx, ComputeBackend backend); - std::vector decode_tdt_tokens_with_state( - CactusGraph* gb, - size_t encoder_hidden_node, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState* stream_state, - size_t* out_confirmed_count = nullptr, - double* out_raw_decoder_time_ms = nullptr) const; - std::vector greedy_decode_tdt_tokens(CactusGraph* gb, size_t encoder_hidden_node) const; - - struct WeightNodeIDs { - size_t subsampling_conv0_weight = 0; - size_t subsampling_conv0_bias = 0; - size_t subsampling_depthwise1_weight = 0; - size_t subsampling_depthwise1_bias = 0; - size_t subsampling_pointwise1_weight = 0; - size_t subsampling_pointwise1_bias = 0; - size_t subsampling_depthwise2_weight = 0; - size_t subsampling_depthwise2_bias = 0; - size_t subsampling_pointwise2_weight = 0; - size_t subsampling_pointwise2_bias = 0; - size_t subsampling_linear_weight = 0; - size_t subsampling_linear_bias = 0; - - struct LayerWeights { - size_t ff1_linear1_weight = 0; - size_t ff1_linear1_bias = 0; - size_t ff1_linear2_weight = 0; - size_t ff1_linear2_bias = 0; - - size_t ff2_linear1_weight = 0; - size_t ff2_linear1_bias = 0; - size_t ff2_linear2_weight = 0; - size_t ff2_linear2_bias = 0; - - size_t self_attn_q_weight = 0; - size_t self_attn_q_bias = 0; - size_t self_attn_k_weight = 0; - size_t self_attn_k_bias = 0; - size_t self_attn_v_weight = 0; - size_t self_attn_v_bias = 0; - size_t self_attn_output_weight = 0; - size_t self_attn_output_bias = 0; - size_t self_attn_relative_k_weight = 0; - size_t self_attn_bias_u = 0; - size_t self_attn_bias_v = 0; - - size_t norm_ff1_weight = 0; - size_t norm_ff1_bias = 0; - size_t norm_self_attn_weight = 0; - size_t norm_self_attn_bias = 0; - size_t norm_conv_weight = 0; - size_t norm_conv_bias = 0; - size_t norm_ff2_weight = 0; - size_t norm_ff2_bias = 0; - size_t norm_out_weight = 0; - size_t norm_out_bias = 0; - - size_t conv_pointwise1_weight = 0; - size_t conv_pointwise1_bias = 0; - size_t conv_depthwise_weight = 0; - size_t conv_depthwise_bias = 0; - size_t conv_pointwise2_weight = 0; - size_t conv_pointwise2_bias = 0; - size_t conv_batchnorm_weight = 0; - size_t conv_batchnorm_bias = 0; - size_t conv_batchnorm_running_mean = 0; - size_t conv_batchnorm_running_var = 0; - }; - - struct PredictorLayerWeights { - size_t weight_ih = 0; - size_t weight_hh = 0; - size_t bias = 0; - }; - - size_t predictor_embed = 0; - std::vector predictor_layers; - size_t joint_enc_weight = 0; - size_t joint_enc_bias = 0; - size_t joint_pred_weight = 0; - size_t joint_pred_bias = 0; - size_t joint_out_weight = 0; - size_t joint_out_bias = 0; - - std::vector layers; - } weight_nodes_; - - bool tdt_tokens_ready_ = false; - size_t tdt_emit_index_ = 0; - std::vector tdt_tokens_; - size_t last_input_token_count_ = 0; - - std::unique_ptr npu_encoder_; - bool use_npu_encoder_ = false; - bool has_cpu_encoder_weights_ = false; -}; - - -class Lfm2VlModel : public Model { -public: - Lfm2VlModel(); - explicit Lfm2VlModel(const Config& config); - ~Lfm2VlModel() override = default; - - bool init(const std::string& model_folder, size_t context_size, const std::string& system_prompt = "", bool do_warmup = true) override; - size_t forward(const std::vector& tokens, bool use_cache = false) override; - - uint32_t decode(const std::vector& tokens, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - - void prefill(const std::vector& tokens, size_t chunk_size = 256, const std::string& profile_file = "") override; - - void prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file = "") override; - - uint32_t decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature = -1.0f, - float top_p = -1.0f, - size_t top_k = 0, - const std::string& profile_file = "", - float* out_entropy = nullptr, - float min_p = 0.15f, - float repetition_penalty = 1.1f) override; - - void reset_cache() override; - std::vector get_image_embeddings(const std::string& image_path) override; - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override; - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - struct ProjectedTileFeature { - size_t node_id; - size_t token_count; - }; - - struct TextEmbeddingInput { - size_t input_node; - std::vector tokens; - }; - - struct MergedEmbeddingResult { - size_t node_id; - size_t seq_len; - }; - - struct ForwardImageResult { - size_t final_hidden_node; - size_t seq_len; - }; - std::vector get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend); - - ForwardImageResult forward_images( - CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache); - size_t build_multimodal_projector( - CactusGraph* gb, - size_t image_features, - size_t tile_h, - size_t tile_w, - ComputeBackend backend); - size_t pixel_unshuffle(CactusGraph* gb, size_t hidden_states, size_t height, size_t width, size_t channels); - MergedEmbeddingResult merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs); - Siglip2VisionModel vision_tower_; - LFM2Model language_model_; - Siglip2Preprocessor preprocessor_; - struct ProjectorWeights { - size_t layer_norm_weight; - size_t layer_norm_bias; - size_t linear_1_weight; - size_t linear_1_bias; - size_t linear_2_weight; - size_t linear_2_bias; - } projector_weights_; - - bool vision_weights_loaded_ = false; - bool language_weights_loaded_ = false; - - bool image_prefill_completed_ = false; - size_t last_token_count_ = 0; -}; - -class SileroVADModel : public Model { -public: - static constexpr size_t CONTEXT_SIZE = 64; - static constexpr size_t CHUNK_SIZE = 512; - static constexpr size_t REFLECT_PAD_SIZE = 64; - static constexpr size_t HIDDEN_SIZE = 128; - static constexpr size_t GATE_SIZE = 512; - - SileroVADModel(); - explicit SileroVADModel(const Config& config); - ~SileroVADModel() override; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - struct SpeechTimestamp { - size_t start; - size_t end; - }; - - struct SpeechTimestampsOptions { - float threshold = 0.5f; - float neg_threshold = 0.0f; - int min_speech_duration_ms = 250; - float max_speech_duration_s = std::numeric_limits::infinity(); - int min_silence_duration_ms = 100; - int speech_pad_ms = 30; - int window_size_samples = 512; - int min_silence_at_max_speech = 98; - bool use_max_poss_sil_at_max_speech = true; - int sampling_rate = 16000; - }; - - float process_chunk(const std::vector& audio_chunk); - void reset_states(); - std::vector get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("SileroVAD: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("SileroVAD: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("SileroVAD: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("SileroVAD: use process_chunk() instead"); - } - - size_t forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache = false) override; - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(); - - struct VADGraphNodes { - size_t input; - size_t h_prev; - size_t c_prev; - size_t h_new; - size_t c_new; - size_t output; - size_t lstm_output; - size_t encoder_output; - } graph_nodes_; - - struct VADWeightNodes { - size_t stft_basis; - struct EncoderBlock { - size_t conv_weight; - size_t conv_bias; - }; - std::vector encoder_blocks; - size_t lstm_weight_ih; - size_t lstm_weight_hh; - size_t lstm_bias_ih; - size_t lstm_bias_hh; - size_t output_conv_weight; - size_t output_conv_bias; - } weight_nodes_; - - struct VADState { - std::vector<__fp16> h; - std::vector<__fp16> c; - std::vector context; - std::vector input_buf; - std::vector<__fp16> input_fp16; - } state_; - - CactusGraph graph_; - std::string weights_path_; -}; - -class YoutuModel : public Model { -public: - YoutuModel(); - explicit YoutuModel(const Config& config); - ~YoutuModel() override = default; - - void reset_cache() override; - -protected: - size_t build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const override; - - size_t build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache = false, size_t position_offset = 0) override; - - size_t forward(const std::vector& tokens, bool use_cache = false) override; - void load_weights_to_graph(CactusGraph* gb) override; - std::vector get_kv_layer_dims() const override; - void post_init() override; - void post_execute_updates(CactusGraph* gb, size_t seq_len) override; - -private: - KVCache v_cache_; - std::vector cache_v_nodes_; - - struct WeightNodeIDs { - size_t output_weight = 0; - size_t output_norm_weight = 0; - - struct LayerWeights { - size_t attn_q_weight = 0; - size_t attn_q_a_weight = 0; - size_t attn_q_a_norm_weight = 0; - size_t attn_q_b_weight = 0; - size_t attn_kv_a_weight = 0; - size_t attn_kv_a_norm_weight = 0; - size_t attn_kv_b_weight = 0; - size_t attn_output_weight = 0; - size_t input_layernorm_weight = 0; - size_t post_attention_layernorm_weight = 0; - size_t ffn_gate_weight = 0; - size_t ffn_up_weight = 0; - size_t ffn_down_weight = 0; - size_t attn_q_a_bias = 0; - size_t attn_kv_a_bias = 0; - size_t attn_output_bias = 0; - }; - - std::vector layers; - } weight_nodes_; -}; - -class PyAnnoteModel : public Model { -public: - PyAnnoteModel(); - explicit PyAnnoteModel(const Config& config); - ~PyAnnoteModel() override = default; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - std::vector diarize(const float* pcm_f32, size_t num_samples, size_t step_samples = 16000, bool raw_powerset = false); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("PyAnnote: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("PyAnnote: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("PyAnnote: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("PyAnnote: use diarize() instead"); - } - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(); - - struct LSTMLayerWeights { - size_t w_ih_fwd, w_hh_fwd, b_ih_fwd, b_hh_fwd; - size_t w_ih_bwd, w_hh_bwd, b_ih_bwd, b_hh_bwd; - }; - - struct WeightNodeIDs { - size_t sinc_filters; - size_t wav_norm_weight, wav_norm_bias; - size_t norm0_weight, norm0_bias; - size_t conv1_weight, conv1_bias; - size_t norm1_weight, norm1_bias; - size_t conv2_weight, conv2_bias; - size_t norm2_weight, norm2_bias; - LSTMLayerWeights lstm_layers[4]; - size_t linear0_weight, linear0_bias; - size_t linear1_weight, linear1_bias; - size_t classifier_weight, classifier_bias; - } weight_nodes_; - - CactusGraph graph_; - size_t audio_input_ = 0; - size_t output_node_ = 0; - std::vector hamming_; - std::vector<__fp16> chunk_buf_; -}; - -class WeSpeakerModel : public Model { -public: - WeSpeakerModel(); - explicit WeSpeakerModel(const Config& config); - ~WeSpeakerModel() override = default; - - bool init(const std::string& model_folder, size_t context_size = 0, - const std::string& system_prompt = "", bool do_warmup = false) override; - - std::vector embed(const float* fbank_features, size_t num_features); - std::vector embed(const float* fbank_features, size_t num_features, - const float* mask_weights, size_t mask_num_frames); - -protected: - size_t build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("WeSpeaker: build_attention unused"); - } - - size_t build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const override { - throw std::runtime_error("WeSpeaker: build_mlp unused"); - } - - size_t build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) override { - throw std::runtime_error("WeSpeaker: build_transformer_block unused"); - } - - size_t forward(const std::vector&, bool = false) override { - throw std::runtime_error("WeSpeaker: use embed() instead"); - } - - void load_weights_to_graph(CactusGraph* gb) override; - -private: - void build_graph(size_t num_frames); - void build_graph_masked(size_t num_frames); - - struct ResBlockWeights { - size_t conv1_w, conv1_b; - size_t conv2_w, conv2_b; - size_t shortcut_conv_w, shortcut_conv_b; - bool has_shortcut = false; - }; - - ResBlockWeights load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut); - size_t build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2); - size_t build_resnet_body(CactusGraph* gb, size_t x); - - struct WeightNodeIDs { - size_t conv1_w, conv1_b; - std::vector layer1, layer2, layer3, layer4; - size_t seg1_w, seg1_b; - } weight_nodes_; - - CactusGraph graph_; - size_t audio_input_ = 0; - size_t mask_input_ = 0; - size_t output_node_ = 0; - size_t current_num_frames_ = 0; - bool current_masked_ = false; - std::vector<__fp16> input_buf_; - std::vector mask_buf_; -}; - -} -} - -#include "gemma4/model_gemma4.h" diff --git a/cactus/models/model_gemma.cpp b/cactus/models/model_gemma.cpp deleted file mode 100644 index a128c8905..000000000 --- a/cactus/models/model_gemma.cpp +++ /dev/null @@ -1,192 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include - - -namespace cactus { -namespace engine { - -GemmaModel::GemmaModel() : Model() {} - -GemmaModel::GemmaModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void GemmaModel::post_init() { - kv_cache_.set_window_size(0, 0); -} - -void GemmaModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(layer_prefix + "post_ffn_norm.weights"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t GemmaModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - bool is_global_attention = ((layer_idx + 1) % 6) == 0; - - if (config_.rope_theta > 0) { - float rope_freq = is_global_attention ? 1000000.0f : 10000.0f; - - q_proj_4d = gb->rope(q_proj_4d, rope_freq, position_offset); - k_proj_4d = gb->rope(k_proj_4d, rope_freq, position_offset); - } - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - size_t window_size = is_global_attention ? 0 : 512; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim, window_size - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset, window_size); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - - -size_t GemmaModel::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t gate_output = gb->matmul(input, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(input, layer.ffn_up_weight, true, backend); - size_t gate_gelu = gb->gelu(gate_output); - size_t gated = gb->multiply(gate_gelu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - -size_t GemmaModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - - auto normalized_attn = gb->rms_norm(attn_output, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - - auto after_attention = gb->add_clipped(hidden, normalized_attn); - - auto pre_mlp_norm = gb->rms_norm(after_attention, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, pre_mlp_norm, layer_idx, backend); - - auto normalized_mlp = gb->rms_norm(mlp_output, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - return gb->add_clipped(after_attention, normalized_mlp); -} - - -size_t GemmaModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto seq_len = static_cast(tokens.size()); - - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - float embed_scale = std::sqrt(static_cast(config_.hidden_dim)); - hidden = gb->scalar_multiply(hidden, embed_scale); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} - -} -} \ No newline at end of file diff --git a/cactus/models/model_gemma3n.cpp b/cactus/models/model_gemma3n.cpp deleted file mode 100644 index 65a3b8173..000000000 --- a/cactus/models/model_gemma3n.cpp +++ /dev/null @@ -1,475 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -static const float RSQRT2 = 1.0f / std::sqrt(2.0f); - -GemmaModel3n::GemmaModel3n() : Model() {} - -GemmaModel3n::GemmaModel3n(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -std::vector GemmaModel3n::get_kv_layer_dims() const { - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (n > num_shared) ? n - num_shared : n; - - std::vector dims(n); - for (uint32_t i = 0; i < n; i++) { - dims[i] = (i >= first_shared) ? 0 : config_.attention_head_dim; - } - return dims; -} - -void GemmaModel3n::post_init() { - kv_cache_.set_window_size(0, 0); - - uint32_t n = config_.num_layers; - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (n > num_shared) ? n - num_shared : n; - - kv_share_map_.resize(n, -1); - shared_k_nodes_.resize(n, 0); - shared_v_nodes_.resize(n, 0); - - auto is_global_layer = [&](uint32_t idx) -> bool { - if (!config_.layer_types.empty() && idx < config_.layer_types.size()) { - const auto& lt = config_.layer_types[idx]; - return (lt == "global" || lt == "full" || lt == "full_attention"); - } - return (idx % 5) == 4; - }; - - for (uint32_t i = first_shared; i < n; i++) { - bool is_global = is_global_layer(i); - for (int j = static_cast(first_shared) - 1; j >= 0; j--) { - if (is_global_layer(j) == is_global) { - kv_share_map_[i] = j; - break; - } - } - } -} - -void GemmaModel3n::load_weights_to_graph(CactusGraph* gb) { - assert(config_.altup_num_inputs == 4 && "WeightNodeIDs altup arrays assume exactly 4 AltUp streams"); - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (int i = 0; i < 3; i++) { - auto idx = std::to_string(i); - weight_nodes_.altup_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_proj_" + idx + ".weights"); - weight_nodes_.altup_unembed_proj_weights[i] = gb->mmap_weights(model_folder_path_ + "/altup_unembed_proj_" + idx + ".weights"); - } - - weight_nodes_.embed_tokens_per_layer = gb->mmap_embeddings(model_folder_path_ + "/embed_tokens_per_layer.weights"); - weight_nodes_.per_layer_model_proj = gb->mmap_weights(model_folder_path_ + "/per_layer_model_proj.weights"); - weight_nodes_.per_layer_proj_norm = gb->mmap_weights(model_folder_path_ + "/per_layer_proj_norm.weights"); - - uint32_t num_shared = config_.num_kv_shared_layers; - uint32_t first_shared = (config_.num_layers > num_shared) ? config_.num_layers - num_shared : config_.num_layers; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_shared = (i >= first_shared); - - layer.attn_q_weight = gb->mmap_weights(prefix + "attn_q.weights"); - layer.attn_k_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k.weights"); - layer.attn_v_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = is_shared ? 0 : gb->mmap_weights(prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(prefix + "post_attn_norm.weights"); - layer.pre_feedforward_layernorm_weight = gb->mmap_weights(prefix + "pre_ffn_norm.weights"); - layer.post_feedforward_layernorm_weight = gb->mmap_weights(prefix + "post_ffn_norm.weights"); - - layer.altup_router_norm = gb->mmap_weights(prefix + "altup_router_norm.weights"); - layer.altup_prediction_coefs = gb->mmap_weights(prefix + "altup_prediction_coefs.weights"); - layer.altup_correction_coefs = gb->mmap_weights(prefix + "altup_correction_coefs.weights"); - layer.altup_correct_output_scale = gb->mmap_weights(prefix + "altup_correct_output_scale.weights"); - layer.altup_modality_router = gb->mmap_weights(prefix + "altup_modality_router.weights"); - layer.laurel_left = gb->mmap_weights(prefix + "laurel_left.weights"); - layer.laurel_right = gb->mmap_weights(prefix + "laurel_right.weights"); - layer.laurel_norm = gb->mmap_weights(prefix + "laurel_norm.weights"); - layer.per_layer_gate = gb->mmap_weights(prefix + "per_layer_gate.weights"); - layer.per_layer_proj = gb->mmap_weights(prefix + "per_layer_proj.weights"); - layer.post_per_layer_norm = gb->mmap_weights(prefix + "post_per_layer_norm.weights"); - } - - size_t head_dim = config_.attention_head_dim; - v_norm_ones_weight_.assign(head_dim, static_cast<__fp16>(1.0f)); - v_norm_ones_node_ = gb->input({head_dim}, Precision::FP16); - gb->set_external_input(v_norm_ones_node_, v_norm_ones_weight_.data(), Precision::FP16); -} - - -size_t GemmaModel3n::build_rms_norm_no_weight(CactusGraph* gb, size_t input, size_t num_rows, size_t row_dim) const { - auto flat = gb->reshape(input, {num_rows, row_dim}); - return gb->rms_norm(flat, v_norm_ones_node_, config_.layer_norm_eps); -} - -size_t GemmaModel3n::build_magnitude_normalize(CactusGraph* gb, size_t reference, size_t target) const { - size_t rows = gb->get_output_buffer(reference).shape[0]; - auto ref_sq = gb->reshape(gb->mean(gb->multiply(reference, reference), 1), {rows, 1}); - auto tgt_sq = gb->reshape(gb->mean(gb->multiply(target, target), 1), {rows, 1}); - auto ref_mag = gb->scalar_sqrt(ref_sq); - auto tgt_mag = gb->scalar_sqrt(gb->scalar_add(tgt_sq, 1e-5f)); - auto ratio = gb->divide(ref_mag, tgt_mag); - return gb->multiply(target, ratio); -} - -size_t GemmaModel3n::build_gaussian_topk(CactusGraph* gb, size_t input, float ppf) const { - return gb->gaussian_topk(input, ppf); -} - -size_t GemmaModel3n::build_laurel(CactusGraph* gb, size_t normed_input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto x = gb->matmul(normed_input, layer.laurel_left, true, backend); - x = gb->matmul(x, layer.laurel_right, true, backend); - return gb->add(normed_input, gb->rms_norm(x, layer.laurel_norm, config_.layer_norm_eps)); -} - -size_t GemmaModel3n::build_altup_router_modalities(CactusGraph* gb, size_t stream0, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto x = gb->rms_norm(stream0, layer.altup_router_norm, config_.layer_norm_eps); - x = gb->scalar_multiply(x, 1.0f / static_cast(config_.hidden_dim)); - return gb->tanh(gb->matmul(x, layer.altup_modality_router, true, backend)); -} - -void GemmaModel3n::build_altup_predict(CactusGraph* gb, size_t modalities, uint32_t layer_idx, - const size_t* streams, size_t* predictions) const { - uint32_t n = config_.altup_num_inputs; - auto coefs = gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_prediction_coefs, true, ComputeBackend::CPU); - - size_t seq_len = gb->get_output_buffer(streams[0]).shape[0]; - auto fused = gb->altup_predict(coefs, streams, n); - for (uint32_t i = 0; i < n; i++) { - predictions[i] = gb->slice(fused, 0, i * seq_len, seq_len); - } -} - -void GemmaModel3n::build_altup_correct(CactusGraph* gb, size_t activated, size_t modalities, uint32_t layer_idx, - ComputeBackend backend, const size_t* predictions, size_t* corrected) const { - uint32_t n = config_.altup_num_inputs; - auto coefs = gb->scalar_add(gb->matmul(modalities, weight_nodes_.layers[layer_idx].altup_correction_coefs, true, backend), 1.0f); - auto innovation = gb->subtract(activated, predictions[0]); - - size_t seq_len = gb->get_output_buffer(predictions[0]).shape[0]; - auto fused = gb->altup_correct(coefs, innovation, predictions, n); - for (uint32_t i = 0; i < n; i++) { - corrected[i] = gb->slice(fused, 0, i * seq_len, seq_len); - } -} - -void GemmaModel3n::build_per_layer_input(CactusGraph* gb, size_t pli_combined, uint32_t layer_idx, - ComputeBackend backend, size_t* streams) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - - auto gated = gb->multiply(streams[0], layer.altup_correct_output_scale); - gated = gb->gelu(gb->matmul(gated, layer.per_layer_gate, true, backend)); - - auto pli = gb->multiply(gated, gb->slice(pli_combined, 1, layer_idx * pli_dim, pli_dim)); - pli = gb->rms_norm(gb->matmul(pli, layer.per_layer_proj, true, backend), layer.post_per_layer_norm, config_.layer_norm_eps); - - for (uint32_t i = 1; i < config_.altup_num_inputs; i++) - streams[i] = gb->add(streams[i], pli); -} - - -size_t GemmaModel3n::build_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t seq_len = gb->get_output_buffer(input).shape[0]; - size_t num_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - int share_src = (layer_idx < kv_share_map_.size()) ? kv_share_map_[layer_idx] : -1; - - bool is_global = false; - if (!config_.layer_types.empty() && layer_idx < config_.layer_types.size()) { - const auto& lt = config_.layer_types[layer_idx]; - is_global = (lt == "global" || lt == "full" || lt == "full_attention"); - } else { - is_global = (layer_idx % 5) == 4; - } - float rope_freq = is_global ? config_.rope_theta : config_.rope_local_base_freq; - size_t window = is_global ? 0 : config_.sliding_window; - - auto q = gb->matmul(input, layer.attn_q_weight, true, backend); - q = gb->reshape(q, {seq_len * num_heads, head_dim}); - q = gb->rms_norm(q, layer.attn_q_norm_weight, config_.layer_norm_eps); - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - auto q4 = gb->rope(q, rope_freq, position_offset); - - size_t k4, v4; - if (share_src >= 0 && shared_k_nodes_[share_src] != 0) { - k4 = shared_k_nodes_[share_src]; - v4 = shared_v_nodes_[share_src]; - } else { - auto k = gb->matmul(input, layer.attn_k_weight, true, backend); - k = gb->reshape(k, {seq_len * kv_heads, head_dim}); - k = gb->rms_norm(k, layer.attn_k_norm_weight, config_.layer_norm_eps); - k = gb->reshape(k, {1, seq_len, kv_heads, head_dim}); - k4 = gb->rope(k, rope_freq, position_offset); - - auto v_proj = gb->matmul(input, layer.attn_v_weight, true, backend); - auto v = build_rms_norm_no_weight(gb, v_proj, seq_len * kv_heads, head_dim); - v4 = gb->reshape(v, {1, seq_len, kv_heads, head_dim}); - - shared_k_nodes_[layer_idx] = k4; - shared_v_nodes_[layer_idx] = v4; - } - - if (use_cache && share_src < 0) { - cache_k_output_nodes_[layer_idx] = k4; - cache_v_output_nodes_[layer_idx] = v4; - } - - size_t cache_src = (share_src >= 0) ? static_cast(share_src) : layer_idx; - size_t attn; - if (use_cache && !kv_cache_.is_empty()) { - attn = gb->attention_int8_hybrid( - q4, k4, v4, attention_scale_, position_offset, - kv_cache_.get_keys_int8(cache_src), kv_cache_.get_values_int8(cache_src), - kv_cache_.get_key_scales(cache_src), kv_cache_.get_value_scales(cache_src), - kv_cache_.current_seq_len, kv_heads, head_dim, window); - } else { - attn = gb->attention(q4, k4, v4, attention_scale_, position_offset, window); - } - - auto o_proj = gb->matmul(gb->reshape(attn, {seq_len, num_heads * head_dim}), layer.attn_output_weight, true, backend); - return o_proj; -} - - -size_t GemmaModel3n::build_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto gate = gb->matmul(input, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(input, layer.ffn_up_weight, true, backend); - - if (layer_idx < config_.activation_sparsity_ppf.size() && config_.activation_sparsity_ppf[layer_idx] > 0.0f) - gate = build_gaussian_topk(gb, gate, config_.activation_sparsity_ppf[layer_idx]); - - auto gate_activated = gb->gelu(gate); - auto activated = gb->multiply(gate_activated, up); - return gb->matmul(activated, layer.ffn_down_weight, true, backend); -} - - -size_t GemmaModel3n::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normed = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - - auto laurel = build_laurel(gb, normed, layer_idx, backend); - auto attn_raw = build_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto attn = gb->rms_norm(attn_raw, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto combined = gb->add(gb->add(hidden, attn), laurel); - auto residual = gb->scalar_multiply(combined, RSQRT2); - - auto pre_mlp = gb->rms_norm(residual, layer.pre_feedforward_layernorm_weight, config_.layer_norm_eps); - auto mlp_raw = build_mlp(gb, pre_mlp, layer_idx, backend); - auto mlp = gb->rms_norm(mlp_raw, layer.post_feedforward_layernorm_weight, config_.layer_norm_eps); - - auto block_out = gb->add(residual, mlp); - return block_out; -} - - -size_t GemmaModel3n::build_preamble(CactusGraph* gb, size_t seq_len, ComputeBackend backend, - size_t& token_input, size_t& pli_input, size_t* streams) { - uint32_t num_layers = config_.num_layers; - uint32_t pli_dim = config_.hidden_size_per_layer_input; - uint32_t num_altup = config_.altup_num_inputs; - - token_input = gb->input({seq_len}, Precision::FP32); - auto x = gb->scalar_multiply(gb->embedding(embedding_node_id_, token_input), - std::sqrt(static_cast(config_.hidden_dim))); - - pli_input = gb->input({seq_len}, Precision::FP32); - auto pli_embed = gb->scalar_multiply(gb->embedding(weight_nodes_.embed_tokens_per_layer, pli_input), - std::sqrt(static_cast(pli_dim))); - auto pli_proj = gb->scalar_multiply(gb->matmul(x, weight_nodes_.per_layer_model_proj, true, backend), - 1.0f / std::sqrt(static_cast(config_.hidden_dim))); - pli_proj = gb->reshape(pli_proj, {seq_len * num_layers, pli_dim}); - pli_proj = gb->rms_norm(pli_proj, weight_nodes_.per_layer_proj_norm, config_.layer_norm_eps); - pli_proj = gb->reshape(pli_proj, {seq_len, num_layers * pli_dim}); - auto pli_combined = gb->scalar_multiply(gb->add(pli_proj, pli_embed), RSQRT2); - - streams[0] = x; - for (uint32_t i = 1; i < num_altup; i++) { - streams[i] = build_magnitude_normalize(gb, x, gb->matmul(x, weight_nodes_.altup_proj_weights[i - 1], true, backend)); - } - - return pli_combined; -} - -void GemmaModel3n::build_layer(CactusGraph* gb, uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t pos_offset, size_t pli, size_t* streams) { - auto modalities = build_altup_router_modalities(gb, streams[0], layer_idx, backend); - - size_t predictions[4]; - build_altup_predict(gb, modalities, layer_idx, streams, predictions); - - auto activated = build_transformer_block(gb, predictions[0], layer_idx, backend, use_cache, pos_offset); - - auto modalities_post = build_altup_router_modalities(gb, activated, layer_idx, backend); - - build_altup_correct(gb, activated, modalities_post, layer_idx, backend, predictions, streams); - - if (config_.hidden_size_per_layer_input > 0) { - build_per_layer_input(gb, pli, layer_idx, backend, streams); - } -} - -size_t GemmaModel3n::build_output_head(CactusGraph* gb, size_t* streams, ComputeBackend backend) { - uint32_t num_altup = config_.altup_num_inputs; - - for (uint32_t i = 1; i < num_altup; i++) { - streams[i] = gb->matmul(streams[i], weight_nodes_.altup_unembed_proj_weights[i - 1], true, backend); - streams[i] = build_magnitude_normalize(gb, streams[0], streams[i]); - } - - auto hidden = streams[0]; - for (uint32_t i = 1; i < num_altup; i++) - hidden = gb->add(hidden, streams[i]); - hidden = gb->scalar_multiply(hidden, 1.0f / static_cast(num_altup)); - - auto final_normed = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - return final_normed; -} - -void GemmaModel3n::set_token_inputs(CactusGraph* gb, size_t token_input, size_t pli_input, - const std::vector& tokens) { - std::vector input_data(tokens.size()); - for (size_t i = 0; i < tokens.size(); i++) - input_data[i] = static_cast(tokens[i]); - gb->set_input(token_input, input_data.data(), Precision::FP32); - gb->set_input(pli_input, input_data.data(), Precision::FP32); -} - - -size_t GemmaModel3n::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (tokens.empty()) - throw std::runtime_error("Token sequence cannot be empty"); - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - - size_t token_input, pli_input, streams[4]; - auto pli = build_preamble(gb, tokens.size(), backend, token_input, pli_input, streams); - - for (uint32_t i = 0; i < config_.num_layers; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); - - set_token_inputs(gb, token_input, pli_input, tokens); - return build_output_head(gb, streams, backend); -} - - -size_t GemmaModel3n::forward_split(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t seq_len = tokens.size(); - size_t pos_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - uint32_t num_layers = config_.num_layers; - uint32_t first_shared = num_layers - config_.num_kv_shared_layers; - - size_t token_input, pli_input, streams[4]; - auto pli = build_preamble(gb, seq_len, backend, token_input, pli_input, streams); - - for (uint32_t i = 0; i < first_shared; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli, streams); - - size_t hidden_dim = config_.hidden_dim; - for (uint32_t i = 0; i < config_.altup_num_inputs; i++) { - streams[i] = gb->index(streams[i], seq_len - 1, 0); - streams[i] = gb->reshape(streams[i], {1, hidden_dim}); - } - - auto pli_last = gb->index(pli, seq_len - 1, 0); - pli_last = gb->reshape(pli_last, {1, num_layers * config_.hidden_size_per_layer_input}); - - for (uint32_t i = first_shared; i < num_layers; i++) - build_layer(gb, i, backend, use_cache, pos_offset, pli_last, streams); - - set_token_inputs(gb, token_input, pli_input, tokens); - return build_output_head(gb, streams, backend); -} - - -void GemmaModel3n::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (tokens.empty()) - return; - - static constexpr size_t SPLIT_PREFILL_MIN_TOKENS = 32; - bool use_split = config_.num_kv_shared_layers > 0 - && tokens.size() >= SPLIT_PREFILL_MIN_TOKENS - && !std::getenv("CACTUS_DISABLE_SPLIT_PREFILL"); - - if (!use_split) { - Model::prefill(tokens, chunk_size, profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - - auto process_chunk = [&](const std::vector& chunk) { - forward_split(chunk, true); - gb->execute(profile_file); - update_kv_cache(gb, chunk.size()); - }; - - if (tokens.size() <= chunk_size) { - process_chunk(tokens); - return; - } - - size_t num_full_chunks = (tokens.size() - 1) / chunk_size; - for (size_t i = 0; i < num_full_chunks; ++i) { - size_t start = i * chunk_size; - std::vector chunk(tokens.begin() + start, tokens.begin() + start + chunk_size); - if (i == 1) - gb->set_prefill_mode(true); - process_chunk(chunk); - } - - gb->set_prefill_mode(false); - size_t final_start = num_full_chunks * chunk_size; - std::vector final_chunk(tokens.begin() + final_start, tokens.end()); - process_chunk(final_chunk); -} - -} -} diff --git a/cactus/models/model_lfm2.cpp b/cactus/models/model_lfm2.cpp deleted file mode 100644 index 12d10cad9..000000000 --- a/cactus/models/model_lfm2.cpp +++ /dev/null @@ -1,389 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include -#include -#include -namespace cactus { -namespace engine { -LFM2Model::LFM2Model() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -LFM2Model::LFM2Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -void LFM2Model::post_init() { - if (config_.conv_L_cache > 0) { - Precision cache_precision = Precision::FP16; - size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; - conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); - - } - last_forward_used_cache_ = false; - -} -void LFM2Model::reset_cache() { - Model::reset_cache(); - - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } -} -bool LFM2Model::is_cache_empty() const { - return kv_cache_.is_empty(); -} -bool LFM2Model::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -bool LFM2Model::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -void LFM2Model::load_weights_to_graph(CactusGraph* gb) { - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - } - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - } - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - - } - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - bool is_conv_layer = false; - if (i < config_.layer_types.size()) { - std::string layer_type = config_.layer_types[i]; - is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); - - } - if (is_conv_layer) { - layer_entry.type = WeightNodeIDs::LayerType::CONV; - layer.conv_in_proj_weight = gb->mmap_weights(layer_prefix + "conv_in_proj.weights"); - layer.conv_out_proj_weight = gb->mmap_weights(layer_prefix + "conv_out_proj.weights"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - - } else { - layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - - } - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - - } - -} -size_t LFM2Model::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); - const auto& in_proj_buf = gb->get_output_buffer(in_proj); - if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { - throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); - } - - const size_t L = in_proj_buf.shape[0]; - const size_t C = in_proj_buf.shape[1] / 3; - - size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); - size_t B = gb->slice(triplet, 1, 0, 1); - size_t Cg = gb->slice(triplet, 1, 1, 1); - size_t X = gb->slice(triplet, 1, 2, 1); - - B = gb->reshape(B, {L, C}); - - Cg = gb->reshape(Cg, {L, C}); - - X = gb->reshape(X, {L, C}); - size_t Bx = gb->multiply(B, X); - if (use_cache) { - conv_cache_bx_nodes_[layer_idx] = Bx; - - } else { - conv_cache_bx_nodes_[layer_idx] = 0; - - } - const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); - size_t K = wbuf.shape.back(); - - size_t conv_w = layer.conv_depthwise_weight; - if (wbuf.shape.size() == 2) { - K = wbuf.shape[1]; - conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); - } else if (wbuf.shape.size() != 3){ - throw std::runtime_error("Unexpected depthwise weight rank"); - } - K = wbuf.shape.back(); - - size_t conv_input_lc = Bx; - if (use_cache && conv_cache_.window_size > 0) { - auto view = conv_cache_.get_window(layer_idx); - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - - } - if (!segments.empty()) { - conv_input_lc = segments[0]; - for (size_t idx = 1; idx < segments.size(); ++idx) { - conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); - - } - conv_input_lc = gb->concat(conv_input_lc, Bx, 0); - - } - } - - const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); - size_t total_len = conv_input_buf.shape[0]; - - size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); - const size_t dilation = 1; - - size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); - size_t start = total_len > L ? total_len - L : 0; - size_t y_slice = gb->slice(y_nlc, 1, start, L); - - size_t y_lc = gb->reshape(y_slice, {L, C}); - size_t gated = gb->multiply(Cg, y_lc); - size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); - - auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); - - auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); - size_t num_kv_heads = config_.attention_kv_heads; - auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); - - auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); - - auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - return projected; -} - -size_t LFM2Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - auto activated = gb->multiply(gb->silu(gate), up); - - auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); - - return down; -} - -size_t LFM2Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - size_t block_output; - if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { - block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); - } else { - if (layer_idx < conv_cache_bx_nodes_.size()) { - conv_cache_bx_nodes_[layer_idx] = 0; - } - block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto after_block = gb->add(hidden, block_output); - auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); - auto block_result = gb->add(after_block, mlp_output); - return block_result; -} - -size_t LFM2Model::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, - ComputeBackend backend, bool use_cache) { - if (seq_len == 0) { - throw std::runtime_error("Sequence length must be greater than zero"); - } - - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - } - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - last_forward_used_cache_ = use_cache; - - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - size_t hidden = input_embeddings; - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} -size_t LFM2Model::forward(CactusGraph* gb, const std::vector& tokens, - ComputeBackend backend, bool use_cache) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - auto seq_len = static_cast(tokens.size()); - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} -size_t LFM2Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - return forward(gb, tokens, backend, use_cache); -} -void LFM2Model::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { - if (conv_cache_bx_nodes_.empty()) { - return; - } - - if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - return; - } - size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { - conv_cache_bx_nodes_[layer_idx] = 0; - continue; - } - size_t bx_node = conv_cache_bx_nodes_[layer_idx]; - if (bx_node != 0) { - conv_cache_.update(gb, layer_idx, bx_node); - } - conv_cache_bx_nodes_[layer_idx] = 0; - } - last_forward_used_cache_ = false; - -} -} -} \ No newline at end of file diff --git a/cactus/models/model_lfm2moe.cpp b/cactus/models/model_lfm2moe.cpp deleted file mode 100644 index e867531bb..000000000 --- a/cactus/models/model_lfm2moe.cpp +++ /dev/null @@ -1,456 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -LFM2MoEModel::LFM2MoEModel() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -LFM2MoEModel::LFM2MoEModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_bx_nodes_.assign(config_.num_layers, 0); -} -void LFM2MoEModel::post_init() { - if (config_.num_dense_layers > config_.num_layers) { - throw std::runtime_error("num_dense_layers cannot be greater than num_layers for LFM2MoEModel"); - } - if (config_.num_experts == 0 || config_.num_experts_per_tok == 0 || config_.moe_intermediate_dim == 0) { - throw std::runtime_error("LFM2MoEModel requires num_experts, num_experts_per_tok, and moe_intermediate_dim to be non-zero"); - } - if (!config_.use_expert_bias || !config_.norm_topk_prob) { - throw std::runtime_error("LFM2MoEModel currently requires use_expert_bias=true and norm_topk_prob=true"); - } - if (config_.num_experts_per_tok > config_.num_experts) { - throw std::runtime_error("num_experts_per_tok cannot be greater than num_experts"); - } - - if (config_.conv_L_cache > 0) { - Precision cache_precision = Precision::FP16; - size_t conv_window = config_.conv_L_cache > 0 ? config_.conv_L_cache - 1 : 0; - conv_cache_.init(config_.num_layers, config_.hidden_dim, conv_window, cache_precision); - - } - last_forward_used_cache_ = false; - -} - -bool LFM2MoEModel::is_dense_layer(uint32_t layer_idx) const { - return layer_idx < config_.num_dense_layers; -} -void LFM2MoEModel::reset_cache() { - Model::reset_cache(); - - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } -} -bool LFM2MoEModel::is_cache_empty() const { - return kv_cache_.is_empty(); -} -bool LFM2MoEModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - if (!Model::init(model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -bool LFM2MoEModel::init(CactusGraph* external_graph, const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - if (!Model::init(external_graph, model_folder, context_size, system_prompt, do_warmup)) { - return false; - } - - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - - } - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - return true; -} -void LFM2MoEModel::load_weights_to_graph(CactusGraph* gb) { - if (weight_nodes_.layers.size() != config_.num_layers) { - weight_nodes_.layers.resize(config_.num_layers); - } - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - } - auto mmap_or_throw = [&](const std::string& path, bool embedding) -> size_t { - size_t node_id = embedding ? gb->mmap_embeddings(path) : gb->mmap_weights(path); - return node_id; - }; - - embedding_node_id_ = mmap_or_throw(embedding_file_path_, true); - weight_nodes_.output_norm_weight = mmap_or_throw(model_folder_path_ + "/output_norm.weights", false); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - - } else { - weight_nodes_.output_weight = mmap_or_throw(model_folder_path_ + "/output_weight.weights", false); - output_weight_node_id_ = weight_nodes_.output_weight; - - } - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - bool is_conv_layer = false; - if (i < config_.layer_types.size()) { - std::string layer_type = config_.layer_types[i]; - is_conv_layer = (layer_type == "conv" || layer_type == "CONV"); - - } - if (is_conv_layer) { - layer_entry.type = WeightNodeIDs::LayerType::CONV; - layer.conv_in_proj_weight = mmap_or_throw(layer_prefix + "conv_in_proj.weights", false); - layer.conv_out_proj_weight = mmap_or_throw(layer_prefix + "conv_out_proj.weights", false); - layer.conv_depthwise_weight = mmap_or_throw(layer_prefix + "conv_depthwise.weights", false); - - } else { - layer_entry.type = WeightNodeIDs::LayerType::ATTENTION; - layer.attn_q_weight = mmap_or_throw(layer_prefix + "attn_q.weights", false); - layer.attn_k_weight = mmap_or_throw(layer_prefix + "attn_k.weights", false); - layer.attn_v_weight = mmap_or_throw(layer_prefix + "attn_v.weights", false); - layer.attn_output_weight = mmap_or_throw(layer_prefix + "attn_output.weights", false); - layer.attn_q_norm_weight = mmap_or_throw(layer_prefix + "attn_q_norm.weights", false); - layer.attn_k_norm_weight = mmap_or_throw(layer_prefix + "attn_k_norm.weights", false); - - } - layer.input_layernorm_weight = mmap_or_throw(layer_prefix + "input_norm.weights", false); - layer.post_attention_layernorm_weight = mmap_or_throw(layer_prefix + "post_attn_norm.weights", false); - if (is_dense_layer(i)) { - layer.ffn_gate_weight = mmap_or_throw(layer_prefix + "ffn_gate.weights", false); - layer.ffn_up_weight = mmap_or_throw(layer_prefix + "ffn_up.weights", false); - layer.ffn_down_weight = mmap_or_throw(layer_prefix + "ffn_down.weights", false); - } else { - layer.moe_router_weight = mmap_or_throw(layer_prefix + "moe_router.weights", false); - layer.moe_expert_bias = mmap_or_throw(layer_prefix + "moe_expert_bias.weights", false); - layer.moe_experts.resize(config_.num_experts); - for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { - auto& expert = layer.moe_experts[expert_idx]; - std::string expert_prefix = layer_prefix + "moe_expert_" + std::to_string(expert_idx) + "_"; - expert.w1_weight = mmap_or_throw(expert_prefix + "w1.weights", false); - expert.w3_weight = mmap_or_throw(expert_prefix + "w3.weights", false); - expert.w2_weight = mmap_or_throw(expert_prefix + "w2.weights", false); - } - } - - } - -} -size_t LFM2MoEModel::build_conv1d(CactusGraph* gb, size_t input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - size_t in_proj = gb->matmul(input, layer.conv_in_proj_weight, true, backend); - const auto& in_proj_buf = gb->get_output_buffer(in_proj); - if (in_proj_buf.shape.size() != 2 || (in_proj_buf.shape[1] % 3) != 0) { - throw std::runtime_error("Conv in_proj output must be [L, 3*C]"); - } - - const size_t L = in_proj_buf.shape[0]; - const size_t C = in_proj_buf.shape[1] / 3; - - size_t triplet = gb->reshape(in_proj, {L, static_cast(3), C}); - size_t B = gb->slice(triplet, 1, 0, 1); - size_t Cg = gb->slice(triplet, 1, 1, 1); - size_t X = gb->slice(triplet, 1, 2, 1); - - B = gb->reshape(B, {L, C}); - - Cg = gb->reshape(Cg, {L, C}); - - X = gb->reshape(X, {L, C}); - size_t Bx = gb->multiply(B, X); - if (use_cache) { - conv_cache_bx_nodes_[layer_idx] = Bx; - - } else { - conv_cache_bx_nodes_[layer_idx] = 0; - - } - const auto& wbuf = gb->get_output_buffer(layer.conv_depthwise_weight); - size_t K = wbuf.shape.back(); - - size_t conv_w = layer.conv_depthwise_weight; - if (wbuf.shape.size() == 2) { - K = wbuf.shape[1]; - conv_w = gb->reshape(conv_w, {wbuf.shape[0], static_cast(1), K}); - } else if (wbuf.shape.size() != 3){ - throw std::runtime_error("Unexpected depthwise weight rank"); - } - K = wbuf.shape.back(); - - size_t conv_input_lc = Bx; - if (use_cache && conv_cache_.window_size > 0) { - auto view = conv_cache_.get_window(layer_idx); - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, C}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, C}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - - } - if (!segments.empty()) { - conv_input_lc = segments[0]; - for (size_t idx = 1; idx < segments.size(); ++idx) { - conv_input_lc = gb->concat(conv_input_lc, segments[idx], 0); - - } - conv_input_lc = gb->concat(conv_input_lc, Bx, 0); - - } - } - - const auto& conv_input_buf = gb->get_output_buffer(conv_input_lc); - size_t total_len = conv_input_buf.shape[0]; - - size_t x_nlc = gb->reshape(conv_input_lc, {static_cast(1), total_len, C}); - const size_t dilation = 1; - - size_t y_nlc = gb->conv1d_causal(x_nlc, conv_w, K, dilation); - size_t start = total_len > L ? total_len - L : 0; - size_t y_slice = gb->slice(y_nlc, 1, start, L); - - size_t y_lc = gb->reshape(y_slice, {L, C}); - size_t gated = gb->multiply(Cg, y_lc); - size_t projected = gb->matmul(gated, layer.conv_out_proj_weight, true, backend); - return projected; -} - -size_t LFM2MoEModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto q_proj_linear = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj_linear = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj_linear = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj_linear).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - auto q_proj_reshaped = gb->reshape(q_proj_linear, {batch_seq * num_heads, head_dim}); - - auto q_proj_norm = gb->rms_norm(q_proj_reshaped, layer.attn_q_norm_weight, config_.layer_norm_eps); - - auto q_proj = gb->reshape(q_proj_norm, {batch_seq, num_heads * head_dim}); - size_t num_kv_heads = config_.attention_kv_heads; - auto k_proj_reshaped = gb->reshape(k_proj_linear, {batch_seq * num_kv_heads, head_dim}); - - auto k_proj_norm = gb->rms_norm(k_proj_reshaped, layer.attn_k_norm_weight, config_.layer_norm_eps); - - auto k_proj = gb->reshape(k_proj_norm, {batch_seq, num_kv_heads * head_dim}); - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj_linear, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - size_t attn_output_4d; - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - auto projected = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - return projected; -} - -size_t LFM2MoEModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - if (is_dense_layer(layer_idx)) { - auto gate = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - auto up = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - auto activated = gb->multiply(gb->silu(gate), up); - auto down = gb->matmul(activated, layer.ffn_down_weight, true, backend); - return down; - } - - auto router_logits = gb->matmul(normalized_h, layer.moe_router_weight, true, backend); - auto routing_probs = gb->sigmoid(router_logits); - auto scores_for_routing = gb->add(routing_probs, layer.moe_expert_bias); - auto topk_indices_and_values = gb->topk(scores_for_routing, config_.num_experts_per_tok); - auto topk_indices = gb->index(topk_indices_and_values, 0, 0); - - std::vector w1_weights, w3_weights, w2_weights; - w1_weights.reserve(config_.num_experts); - w3_weights.reserve(config_.num_experts); - w2_weights.reserve(config_.num_experts); - for (uint32_t expert_idx = 0; expert_idx < config_.num_experts; ++expert_idx) { - const auto& expert = layer.moe_experts[expert_idx]; - w1_weights.push_back(expert.w1_weight); - w3_weights.push_back(expert.w3_weight); - w2_weights.push_back(expert.w2_weight); - } - - return gb->moe_layer( - normalized_h, - routing_probs, - topk_indices, - w1_weights, - w3_weights, - w2_weights, - config_.num_experts, - config_.num_experts_per_tok, - config_.norm_topk_prob, - 1e-6f, - config_.routed_scaling_factor - ); -} - -size_t LFM2MoEModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - size_t block_output; - if (layer_entry.type == WeightNodeIDs::LayerType::CONV) { - block_output = build_conv1d(gb, normalized_input, layer_idx, backend, use_cache); - } else { - if (layer_idx < conv_cache_bx_nodes_.size()) { - conv_cache_bx_nodes_[layer_idx] = 0; - } - block_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto after_block = gb->add(hidden, block_output); - auto normalized_after_block = gb->rms_norm(after_block, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_block, layer_idx, backend); - auto block_result = gb->add(after_block, mlp_output); - return block_result; -} - -size_t LFM2MoEModel::forward(CactusGraph* gb, size_t input_embeddings, size_t seq_len, - ComputeBackend backend, bool use_cache) { - if (seq_len == 0) { - throw std::runtime_error("Sequence length must be greater than zero"); - } - - if (conv_cache_bx_nodes_.size() != config_.num_layers) { - conv_cache_bx_nodes_.assign(config_.num_layers, 0); - - } - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - last_forward_used_cache_ = use_cache; - - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - - } - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - size_t hidden = input_embeddings; - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} -size_t LFM2MoEModel::forward(CactusGraph* gb, const std::vector& tokens, - ComputeBackend backend, bool use_cache) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - auto seq_len = static_cast(tokens.size()); - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - auto final_hidden = forward(gb, hidden, seq_len, backend, use_cache); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - return final_hidden; -} -size_t LFM2MoEModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - return forward(gb, tokens, backend, use_cache); -} -void LFM2MoEModel::post_execute_updates(CactusGraph* gb, size_t /*seq_len*/) { - if (conv_cache_bx_nodes_.empty()) { - return; - } - - if (!last_forward_used_cache_ || conv_cache_.window_size == 0) { - std::fill(conv_cache_bx_nodes_.begin(), conv_cache_bx_nodes_.end(), 0); - - return; - } - size_t layer_count = std::min(conv_cache_bx_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::CONV) { - conv_cache_bx_nodes_[layer_idx] = 0; - continue; - } - size_t bx_node = conv_cache_bx_nodes_[layer_idx]; - if (bx_node != 0) { - conv_cache_.update(gb, layer_idx, bx_node); - } - conv_cache_bx_nodes_[layer_idx] = 0; - } - last_forward_used_cache_ = false; - -} -} -} diff --git a/cactus/models/model_lfm2vl.cpp b/cactus/models/model_lfm2vl.cpp deleted file mode 100644 index c4357ad0b..000000000 --- a/cactus/models/model_lfm2vl.cpp +++ /dev/null @@ -1,607 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Lfm2VlModel::Lfm2VlModel() : Model() { - config_.model_type = Config::ModelType::LFM2; -} - -Lfm2VlModel::Lfm2VlModel(const Config& config) - : Model(config), - vision_tower_(config), - language_model_(config) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config.downsample_factor); - preprocessor_config.min_tiles = static_cast(config.min_tiles); - preprocessor_config.max_tiles = static_cast(config.max_tiles); - preprocessor_config.use_thumbnail = config.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config.max_num_patches); - preprocessor_config.tile_size = static_cast(config.tile_size); - preprocessor_config.max_pixels_tolerance = config.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config.do_image_splitting; - preprocessor_config.rescale_factor = config.rescale_factor; - preprocessor_config.image_mean[0] = config.image_mean; - preprocessor_config.image_mean[1] = config.image_mean; - preprocessor_config.image_mean[2] = config.image_mean; - preprocessor_config.image_std[0] = config.image_std; - preprocessor_config.image_std[1] = config.image_std; - preprocessor_config.image_std[2] = config.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -bool Lfm2VlModel::init(const std::string& model_folder, size_t context_size, const std::string& system_prompt, bool do_warmup) { - - if (!Model::init(model_folder, context_size, system_prompt, false)) { - return false; - } - auto* shared_graph = static_cast(graph_handle_); - if (!shared_graph) { - throw std::runtime_error("Shared graph was not initialized for Lfm2VlModel"); - } - std::string vision_folder = model_folder; - if (!vision_tower_.init(shared_graph, vision_folder, context_size, "", false)) { - throw std::runtime_error("Failed to initialize vision tower"); - } - - vision_weights_loaded_ = true; - if (!language_model_.init(shared_graph, model_folder, context_size, system_prompt, false)) { - throw std::runtime_error("Failed to initialize language model"); - } - - language_model_.output_weight_node_id_ = output_weight_node_id_; - - language_weights_loaded_ = true; - - if (do_warmup) { - std::string warmup_text = system_prompt.empty() ? "Hello" : system_prompt; - auto warmup_tokens = tokenizer_->encode(warmup_text); - language_model_.decode(warmup_tokens, config_.default_temperature, config_.default_top_p, config_.default_top_k, ""); - language_model_.reset_cache(); - } - - return true; -} - -void Lfm2VlModel::reset_cache() { - Model::reset_cache(); - language_model_.reset_cache(); - image_prefill_completed_ = false; - last_token_count_ = 0; -} - -void Lfm2VlModel::load_weights_to_graph(CactusGraph* gb) { - namespace fs = std::filesystem; - fs::path base(model_folder_path_); - - auto resolve_weight = [&](const std::string& primary, const std::string& fallback = "") -> std::string { - fs::path primary_path = base / primary; - if (fs::exists(primary_path)) { - return primary_path.string(); - } - if (!fallback.empty()) { - fs::path fallback_path = base / fallback; - if (fs::exists(fallback_path)) { - return fallback_path.string(); - } - } - return primary_path.string(); - }; - - try { - projector_weights_.layer_norm_weight = gb->mmap_weights(resolve_weight("projector_layer_norm.weights")); - projector_weights_.layer_norm_bias = gb->mmap_weights(resolve_weight("projector_layer_norm.bias.weights")); - } catch (const std::exception&) { - projector_weights_.layer_norm_weight = 0; - projector_weights_.layer_norm_bias = 0; - } - - projector_weights_.linear_1_weight = gb->mmap_weights(resolve_weight("projector_linear_1.weights", "projector_linear1.weights")); - projector_weights_.linear_1_bias = gb->mmap_weights(resolve_weight("projector_linear_1.bias.weights", "projector_linear1.bias.weights")); - projector_weights_.linear_2_weight = gb->mmap_weights(resolve_weight("projector_linear_2.weights", "projector_linear2.weights")); - projector_weights_.linear_2_bias = gb->mmap_weights(resolve_weight("projector_linear_2.bias.weights", "projector_linear2.bias.weights")); - output_weight_node_id_ = gb->mmap_weights(resolve_weight("token_embeddings.weights")); -} - -size_t Lfm2VlModel::pixel_unshuffle(CactusGraph* gb, size_t hidden_states, - size_t height, size_t width, size_t channels) { - - const size_t factor = config_.downsample_factor; - const size_t new_height = height / factor; - const size_t new_width = width / factor; - size_t step1 = gb->reshape(hidden_states, {1, height, new_width, channels * factor}); - step1 = gb->transposeN(step1, {0, 2, 1, 3}); - size_t step2 = gb->reshape(step1, {1, new_width, new_height, channels * factor * factor}); - size_t result = gb->transposeN(step2, {0, 2, 1, 3}); - return result; -} - -size_t Lfm2VlModel::build_multimodal_projector(CactusGraph* gb, size_t image_features, - size_t tile_h, size_t tile_w, ComputeBackend backend) { - const size_t vision_hidden = config_.vision_embed_dim; - - const auto& input_buf = gb->get_output_buffer(image_features); - size_t image_features_fp16 = (input_buf.precision == Precision::FP16) - ? image_features - : gb->precision_cast(image_features, Precision::FP16); - - size_t unshuffled = pixel_unshuffle(gb, image_features_fp16, tile_h, tile_w, vision_hidden); - const size_t factor = config_.downsample_factor; - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t in_channels = vision_hidden * factor * factor; - const size_t seq_len = new_h * new_w; - size_t flattened = gb->reshape(unshuffled, {seq_len, in_channels}); - size_t normalized = flattened; - if (projector_weights_.layer_norm_weight != 0 && projector_weights_.layer_norm_bias != 0) { - normalized = gb->layernorm(flattened, projector_weights_.layer_norm_weight, - projector_weights_.layer_norm_bias, config_.layer_norm_eps); - } - size_t hidden = gb->matmul(normalized, projector_weights_.linear_1_weight, true, backend); - hidden = gb->add(hidden, projector_weights_.linear_1_bias); - hidden = gb->gelu(hidden); - size_t output = gb->matmul(hidden, projector_weights_.linear_2_weight, true, backend); - output = gb->add(output, projector_weights_.linear_2_bias); - return output; -} - -std::vector Lfm2VlModel::get_image_embeddings(const std::string& image_path) { - return vision_tower_.get_image_embedding(image_path); -} - -std::vector Lfm2VlModel::get_image_features( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - size_t vision_output = vision_tower_.forward_vision(gb, preprocessed_image, backend); - std::vector projected_features; - projected_features.reserve(preprocessed_image.spatial_shapes.size()); - - size_t offset = 0; - for (size_t tile_idx = 0; tile_idx < preprocessed_image.spatial_shapes.size(); ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const size_t tile_h = shape.first; - const size_t tile_w = shape.second; - const size_t tile_tokens = tile_h * tile_w; - const size_t factor = config_.downsample_factor; - if (factor == 0) { - throw std::runtime_error("Downsample factor must be greater than zero"); - } - if (tile_h % factor != 0 || tile_w % factor != 0) { - throw std::runtime_error("Tile dimensions must be divisible by downsample factor"); - } - const size_t new_h = tile_h / factor; - const size_t new_w = tile_w / factor; - const size_t projected_tokens = new_h * new_w; - - size_t tile_features = gb->slice(vision_output, 0, offset, tile_tokens); - offset += tile_tokens; - - size_t reshaped = gb->reshape(tile_features, {1, tile_h, tile_w, config_.vision_embed_dim}); - size_t projected = build_multimodal_projector(gb, reshaped, tile_h, tile_w, backend); - ProjectedTileFeature feature{}; - feature.node_id = projected; - feature.token_count = projected_tokens; - projected_features.push_back(feature); - - } - - return projected_features; -} - -Lfm2VlModel::MergedEmbeddingResult Lfm2VlModel::merge_image_text_embeddings( - CactusGraph* gb, - const std::vector& tokens, - const std::vector>& image_embedding_nodes, - std::vector& text_embedding_inputs) { - - text_embedding_inputs.clear(); - - Tokenizer* tokenizer = language_model_.get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer must be initialized before merging embeddings"); - } - - const uint32_t image_token_id = tokenizer->get_image_token_id(); - - auto get_token_id = [tokenizer](const std::string& token_text) -> uint32_t { - auto encoded = tokenizer->encode(token_text); - if (encoded.size() != 1) { - - throw std::runtime_error("Expected single token encoding for " + token_text); - } - return encoded[0]; - }; - - const uint32_t image_start_id = get_token_id("<|image_start|>"); - const uint32_t image_end_id = get_token_id("<|image_end|>"); - - std::vector sequence_nodes; - sequence_nodes.reserve(tokens.size() + image_embedding_nodes.size()); - - std::vector current_segment; - current_segment.reserve(tokens.size()); - - size_t total_seq_len = 0; - - auto flush_segment = [&](void) { - if (current_segment.empty()) { - - return; - } - - const size_t segment_len = current_segment.size(); - TextEmbeddingInput segment; - segment.tokens.swap(current_segment); - segment.input_node = gb->input({segment.tokens.size()}, Precision::FP32); - - const auto& embedding_buffer = gb->get_output_buffer(language_model_.embedding_node_id_); - - for (size_t i = 0; i < embedding_buffer.shape.size(); ++i) { - (void)i; - } - - size_t embedding_node = gb->embedding(language_model_.embedding_node_id_, segment.input_node); - - text_embedding_inputs.push_back(std::move(segment)); - - sequence_nodes.push_back(embedding_node); - - total_seq_len += segment_len; - - current_segment.clear(); - - }; - - size_t token_index = 0; - size_t image_index = 0; - - while (token_index < tokens.size()) { - uint32_t token_id = tokens[token_index]; - - if (token_id == image_start_id) { - flush_segment(); - - if (image_index >= image_embedding_nodes.size()) { - - throw std::runtime_error("Encountered <|image_start|> without corresponding image features"); - } - - current_segment.push_back(token_id); - ++token_index; - - const auto& tiles = image_embedding_nodes[image_index]; - size_t tile_index = 0; - while (token_index < tokens.size()) { - uint32_t inner_token = tokens[token_index]; - if (inner_token == image_token_id) { - flush_segment(); - - if (tile_index >= tiles.size()) { - - throw std::runtime_error("More placeholders than projected tile features"); - } - - const auto& tile = tiles[tile_index++]; - sequence_nodes.push_back(tile.node_id); - - total_seq_len += tile.token_count; - - for (size_t count = 0; count < tile.token_count; ++count) { - if (token_index >= tokens.size()) { - throw std::runtime_error("Insufficient tokens for projected features"); - } - if (tokens[token_index] != image_token_id) { - throw std::runtime_error("Unexpected token encountered within image feature span"); - } - ++token_index; - - } - continue; - } - - current_segment.push_back(inner_token); - - ++token_index; - - if (inner_token == image_end_id) { - flush_segment(); - break; - } - } - - if (tile_index != tiles.size()) { - if (tile_index < tiles.size()) { - for (size_t remaining = tile_index; remaining < tiles.size(); ++remaining) { - (void)tiles[remaining]; - } - } - throw std::runtime_error("Unused projected tile features remain after processing image block"); - } - - ++image_index; - - } else { - current_segment.push_back(token_id); - - ++token_index; - - } - } - - flush_segment(); - if (image_index != image_embedding_nodes.size()) { - throw std::runtime_error("Not all image features were consumed while merging embeddings"); - } - - if (sequence_nodes.empty()) { - throw std::runtime_error("Failed to build embedding sequence from provided tokens"); - } - - size_t merged = sequence_nodes[0]; - for (size_t idx = 1; idx < sequence_nodes.size(); ++idx) { - merged = gb->concat(merged, sequence_nodes[idx], 0); - - } - - return MergedEmbeddingResult{merged, total_seq_len}; -} - -size_t Lfm2VlModel::build_attention(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_attention should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_mlp(CactusGraph*, size_t, uint32_t, ComputeBackend) const { - throw std::runtime_error("build_mlp should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::build_transformer_block(CactusGraph*, size_t, uint32_t, ComputeBackend, bool, size_t) { - throw std::runtime_error("build_transformer_block should not be called directly on Lfm2VlModel"); -} - -size_t Lfm2VlModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t final_hidden = language_model_.forward(gb, tokens, backend, use_cache); - - return final_hidden; -} - -uint32_t Lfm2VlModel::decode(const std::vector& tokens, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); -} - -void Lfm2VlModel::prefill(const std::vector& tokens, size_t chunk_size, const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - - language_model_.prefill(tokens, chunk_size, profile_file); -} - -void Lfm2VlModel::prefill_with_images(const std::vector& tokens, const std::vector& image_paths, - const std::string& profile_file) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (tokens.empty()) { - return; - } - if (image_paths.empty()) { - prefill(tokens, get_prefill_chunk_size(), profile_file); - return; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - gb->execute(profile_file); - - language_model_.post_execute_updates(gb, forward_result.seq_len); - language_model_.update_kv_cache(gb, forward_result.seq_len); - - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); -} - -Lfm2VlModel::ForwardImageResult Lfm2VlModel::forward_images( - CactusGraph* gb, - const std::vector& tokens, - const std::vector& image_paths, - ComputeBackend backend, - bool use_cache) { - if (!gb) { - throw std::runtime_error("Graph must be initialized before forwarding"); - } - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - std::vector> all_image_embeddings; - all_image_embeddings.reserve(image_paths.size()); - for (const auto& image_path : image_paths) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - - auto image_features = get_image_features(gb, preprocessed, backend); - - all_image_embeddings.push_back(std::move(image_features)); - } - - std::vector text_embedding_inputs; - text_embedding_inputs.reserve(tokens.size() / 4 + 1); - - auto merged_embeddings = merge_image_text_embeddings(gb, tokens, all_image_embeddings, text_embedding_inputs); - if (merged_embeddings.seq_len == 0) { - throw std::runtime_error("Merged embedding sequence length cannot be zero"); - } - - for (const auto& embedding_input : text_embedding_inputs) { - if (embedding_input.tokens.empty()) { - continue; - } - - std::vector segment_data(embedding_input.tokens.size()); - for (size_t i = 0; i < embedding_input.tokens.size(); ++i) { - segment_data[i] = static_cast(embedding_input.tokens[i]); - } - gb->set_input(embedding_input.input_node, segment_data.data(), Precision::FP32); - } - size_t final_hidden = language_model_.forward(gb, merged_embeddings.node_id, merged_embeddings.seq_len, backend, use_cache); - return ForwardImageResult{final_hidden, merged_embeddings.seq_len}; -} - -uint32_t Lfm2VlModel::decode_with_images( - const std::vector& tokens, - const std::vector& image_paths, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty) { - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (image_paths.empty()) { - - image_prefill_completed_ = false; - last_token_count_ = tokens.size(); - return language_model_.decode(tokens, temperature, top_p, top_k, profile_file, out_entropy, min_p, repetition_penalty); - } - - if (temperature < 0) { - temperature = config_.default_temperature; - } - if (top_p < 0) { - top_p = config_.default_top_p; - } - if (top_k == 0) { - top_k = config_.default_top_k; - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - bool cache_empty = language_model_.is_cache_empty(); - bool need_prefill = cache_empty || !image_prefill_completed_; - - if (!need_prefill && tokens.size() <= last_token_count_) { - - reset_cache(); - need_prefill = true; - } - - size_t seq_len_for_updates = 0; - size_t final_hidden_node = 0; - - if (need_prefill) { - auto forward_result = forward_images(gb, tokens, image_paths, backend, true); - - final_hidden_node = forward_result.final_hidden_node; - seq_len_for_updates = forward_result.seq_len; - image_prefill_completed_ = true; - last_token_count_ = tokens.size(); - } else { - size_t delta = tokens.size() - last_token_count_; - if (delta > tokens.size()) { - delta = tokens.size(); - } - if (delta == 0) { - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty for cached decode step"); - } - delta = 1; - - } - std::vector incremental_tokens(tokens.end() - delta, tokens.end()); - - final_hidden_node = language_model_.forward(gb, incremental_tokens, backend, true); - - seq_len_for_updates = incremental_tokens.size(); - last_token_count_ = tokens.size(); - } - - auto logits_node_id = gb->matmul(final_hidden_node, language_model_.output_weight_node_id_, true, backend); - size_t sampled_token_id = - language_model_.sample_token(gb, logits_node_id, temperature, top_p, top_k, min_p, repetition_penalty, nullptr); - if (!profile_file.empty()) { - gb->execute(profile_file); - - } else { - gb->execute(); - - } - - compute_entropy(gb, logits_node_id, out_entropy); - - language_model_.post_execute_updates(gb, seq_len_for_updates); - language_model_.update_kv_cache(gb, seq_len_for_updates); - - auto* output_ptr = gb->get_output(sampled_token_id); - uint32_t result = *static_cast(output_ptr); - language_model_.record_sampled_token(result); - return result; -} - -} -} diff --git a/cactus/models/model_moonshine.cpp b/cactus/models/model_moonshine.cpp deleted file mode 100644 index fe03f5983..000000000 --- a/cactus/models/model_moonshine.cpp +++ /dev/null @@ -1,596 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include -namespace cactus { -namespace engine { - -static size_t moonshine_downsampled_len(size_t audio_len) { - if (audio_len < 127) { - return 0; - } - size_t l1 = (audio_len - 127) / 64 + 1; - if (l1 < 7) { - return 0; - } - size_t l2 = (l1 - 7) / 3 + 1; - if (l2 < 3) { - return 0; - } - size_t l3 = (l2 - 1) / 2 + 1; - return l3; -} - -MoonshineModel::MoonshineModel() : Model() {} -MoonshineModel::MoonshineModel(const Config& config) : Model(config) { - weight_nodes_.encoder_layers.resize(config.num_encoder_layers); - weight_nodes_.decoder_layers.resize(config.num_decoder_layers); - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - encoder_block_out_nodes_.resize(config.num_encoder_layers, 0); - encoder_k_persistent_.assign(config.num_decoder_layers, 0); - encoder_v_persistent_.assign(config.num_decoder_layers, 0); -} -void MoonshineModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - if (output_weight_node_id_ == 0) { - output_weight_node_id_ = gb->mmap_weights(model_folder_path_ + "/output_layer.weights"); - } - weight_nodes_.output_weight = output_weight_node_id_; - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } - else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - if (!use_npu_encoder_) { - weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); - weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); - weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); - weight_nodes_.encoder_conv3_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_weight.weights"); - weight_nodes_.encoder_conv3_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv3_bias.bias"); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); - weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); - weight_nodes_.encoder_layer_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); - } - for (uint32_t i = 0; i < config_.num_decoder_layers; i++) { - auto& layer = weight_nodes_.decoder_layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); - layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); - layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); - layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); - layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - if (config_.decoder_act_gelu) { - layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - } - else { - layer.decoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.decoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); - layer.decoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.decoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); - } - layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - } - if (!use_npu_encoder_) { - for (uint32_t i = 0; i < config_.num_encoder_layers; i++) { - auto& layer = weight_nodes_.encoder_layers[i]; - std::string layer_prefix = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; - if (config_.encoder_act_gelu) { - layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - } - else { - layer.encoder_ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.encoder_ffn_gate_bias = gb->mmap_weights(layer_prefix + "ffn_gate.bias"); - layer.encoder_ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.encoder_ffn_up_bias = gb->mmap_weights(layer_prefix + "ffn_up.bias"); - } - layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - } - } -} -static size_t build_encoder_mlp_gelu(CactusGraph* gb, size_t input, size_t w1, size_t b1, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { - auto ffn1_weight = gb->matmul(input, w1, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, b1); - auto ffn1_act = gb->gelu(ffn1_bias); - auto ffn2_weight = gb->matmul(ffn1_act, w2, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, b2); - return ffn2_bias; -} -static size_t build_decoder_mlp_silu(CactusGraph* gb, size_t input, size_t w_gate, size_t b_gate, size_t w_up, size_t b_up, size_t w2, size_t b2, ComputeBackend backend, uint32_t /*layer_idx*/) { - auto gate_weight = gb->matmul(input, w_gate, true, backend); - auto gate_bias = gb->add(gate_weight, b_gate); - auto gate_act = gb->silu(gate_bias); - auto up_weight = gb->matmul(input, w_up, true, backend); - auto up_bias = gb->add(up_weight, b_up); - auto intermediate = gb->multiply(gate_act, up_bias); - auto output_weight = gb->matmul(intermediate, w2, true, backend); - auto output_bias = gb->add(output_weight, b2); - return output_bias; -} -size_t MoonshineModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); - const auto& q_buf = gb->get_output_buffer(q); - if (q_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); - } - size_t T_dec = q_buf.shape[0]; - size_t q_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); - size_t k_4d = 0; - size_t v_4d = 0; - bool is_pop = (encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx])); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".encoder_attn."; - if (is_pop) { - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - else { - size_t enc_norm = last_encoder_post_norm_node_; - size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); - size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); - const auto& k_buf = gb->get_output_buffer(k); - if (k_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); - } - size_t T_enc = k_buf.shape[0]; - k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); - v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); - encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); - } - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); - attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); - size_t output = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); - return output; -} -void MoonshineModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); - cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); -} -void MoonshineModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - encoder_kv_ready_ = false; - first_decode_step_ = true; - encoder_output_host_.clear(); - auto* gb = static_cast(graph_handle_); - if (gb) { - if (last_encoder_post_norm_node_ != 0) { - gb->invalidate_persistent(last_encoder_post_norm_node_); - last_encoder_post_norm_node_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_k_persistent_[i]); - encoder_k_persistent_[i] = 0; - } - if (encoder_v_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_v_persistent_[i]); - encoder_v_persistent_[i] = 0; - } - } - } -} -size_t MoonshineModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t q_proj = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); - size_t k_proj = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); - size_t v_proj = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + ".self_attn."; - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); - } - size_t seq_new = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - size_t num_kv_heads = config_.attention_kv_heads; - size_t q_4d = gb->reshape(q_proj, {1, seq_new, num_heads, head_dim}); - size_t k_4d = gb->reshape(k_proj, {1, seq_new, num_kv_heads, head_dim}); - size_t v_4d = gb->reshape(v_proj, {1, seq_new, num_kv_heads, head_dim}); - size_t rot_dim = std::max(head_dim / 2, (size_t)32); - if (config_.rope_theta > 0) { - q_4d = gb->rope_gptj(q_4d, config_.rope_theta, position_offset, rot_dim); - k_4d = gb->rope_gptj(k_4d, config_.rope_theta, position_offset, rot_dim); - } - size_t final_k = k_4d; - size_t final_v = v_4d; - if (use_cache && !kv_cache_.is_empty()) { - auto k_view = kv_cache_.get_key_view(layer_idx); - auto v_view = kv_cache_.get_value_view(layer_idx); - if (!k_view.ptr1 || !v_view.ptr1) { - throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); - } - size_t cache_len = kv_cache_.current_seq_len; - size_t cache_k_node = gb->input( - {1, cache_len, num_kv_heads, head_dim} -, - kv_cache_.precision - ); - size_t cache_v_node = gb->input( - {1, cache_len, num_kv_heads, head_dim} -, - kv_cache_.precision - ); - if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { - gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); - gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); - } - else { - gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - final_k = gb->concat(cache_k_node, k_4d, 1); - final_v = gb->concat(cache_v_node, v_4d, 1); - } - if (use_cache) { - cache_k_output_nodes_[layer_idx] = final_k; - cache_v_output_nodes_[layer_idx] = final_v; - } - else { - cache_k_output_nodes_[layer_idx] = k_4d; - cache_v_output_nodes_[layer_idx] = v_4d; - } - auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); - auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); - auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); - return output; -} -size_t MoonshineModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.encoder_layers[layer_idx]; - if(use_cache) - throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); - auto q_proj = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); - auto v_proj = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); - auto k_proj = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); - std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + ".self_attn."; - size_t seq_len = gb->get_output_buffer(q_proj).shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - auto q = gb->reshape(q_proj, {1, seq_len, num_heads, head_dim}); - auto k = gb->reshape(k_proj, {1, seq_len, num_heads, head_dim}); - auto v = gb->reshape(v_proj, {1, seq_len, num_heads, head_dim}); - size_t rot_dim = static_cast(head_dim * config_.partial_rotary_factor); - if (rot_dim % 2 != 0) throw std::runtime_error("rot_dim must be even"); - if (config_.rope_theta > 0) { - q = gb->rope_gptj(q, config_.rope_theta, 0, rot_dim); - k = gb->rope_gptj(k, config_.rope_theta, 0, rot_dim); - } - auto attn = gb->attention(q, k, v, attention_scale_, false); - attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); - auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); - return output; -} -size_t MoonshineModel::build_audio_preprocessor(CactusGraph* gb, size_t input) -{ - size_t conv_input = input; - const auto& xbuf = gb->get_output_buffer(input); - if (xbuf.precision == Precision::INT8) { - conv_input = gb->precision_cast(input, Precision::FP16); - } - size_t conv1 = gb->conv1d(conv_input, weight_nodes_.encoder_conv1_weight, 64); - last_conv1_node_ = conv1; - size_t conv1_act = gb->tanh(conv1); - size_t group_norm = gb->groupnorm(conv1_act, weight_nodes_.encoder_norm_weight, weight_nodes_.encoder_norm_bias, 1); - - size_t conv2_with_bias = gb->conv1d_k7s3(group_norm, weight_nodes_.encoder_conv2_weight, weight_nodes_.encoder_conv2_bias); - size_t conv2_act = gb->gelu(conv2_with_bias); - size_t conv3 = gb->conv1d_k3(conv2_act, weight_nodes_.encoder_conv3_weight, 2); - auto bias3_shape = gb->get_output_buffer(weight_nodes_.encoder_conv3_bias).shape; - size_t C3 = bias3_shape[0]; - size_t bias3 = gb->reshape(weight_nodes_.encoder_conv3_bias, {1, C3, 1}); - size_t conv3_with_bias = gb->add(conv3, bias3); - size_t conv3_act = gb->gelu(conv3_with_bias); - const auto& buf = gb->get_output_buffer(conv3_act); - size_t conv_out_transposed; - if (buf.precision == Precision::FP16) { - conv_out_transposed = gb->transpose(conv3_act, ComputeBackend::CPU); - } - else { - size_t conv3_f16 = gb->precision_cast(conv3_act, Precision::FP16); - conv_out_transposed = gb->transpose(conv3_f16, ComputeBackend::CPU); - } - return conv_out_transposed; -} -size_t MoonshineModel::build_encoder_transformer_block( - CactusGraph* gb, - size_t hidden, - uint32_t layer_idx, - ComputeBackend backend, - bool use_cache, - size_t position_offset) -{ - const auto& layer = weight_nodes_.encoder_layers[layer_idx]; - size_t input_layernorm = gb->layernorm( - hidden, - layer.encoder_post_attn_layernorm_weight - ); - size_t self_attn_out = build_encoder_self_attention( - gb, input_layernorm, layer_idx, backend, use_cache, position_offset - ); - size_t post_attention_layernorm_input = gb->add(hidden, self_attn_out); - size_t post_attention_layernorm = gb->layernorm( - post_attention_layernorm_input, - layer.encoder_post_ffn_layernorm_weight - ); - std::string prefix = "model.encoder.layers." + std::to_string(layer_idx) + "."; - size_t mlp_out = build_encoder_mlp_gelu( - gb, post_attention_layernorm, layer.encoder_ffn1_weight, layer.encoder_ffn1_bias, - layer.encoder_ffn2_weight, layer.encoder_ffn2_bias, backend, layer_idx - ); - size_t out = gb->add(post_attention_layernorm_input, mlp_out); - if (layer_idx < encoder_block_out_nodes_.size()) { - encoder_block_out_nodes_[layer_idx] = out; - } - return out; -} -size_t MoonshineModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.decoder_layers[layer_idx]; - size_t input_layernorm = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight); - size_t sa = build_decoder_self_attention(gb, input_layernorm, layer_idx, backend, use_cache, position_offset); - size_t x_post_sa = gb->add(hidden, sa); - size_t post_attention_layernorm = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight); - size_t ca = build_decoder_cross_attention(gb, post_attention_layernorm, layer_idx, backend, use_cache, position_offset); - size_t x_post_ca = gb->add(x_post_sa, ca); - size_t final_layernorm = gb->layernorm(x_post_ca, layer.decoder_post_ffn_layernorm_weight); - size_t mlp_out = build_decoder_mlp_silu( - gb, final_layernorm, layer.decoder_ffn_gate_weight, layer.decoder_ffn_gate_bias, - layer.decoder_ffn_up_weight, layer.decoder_ffn_up_bias, - layer.decoder_ffn2_weight, layer.decoder_ffn2_bias, backend, layer_idx - ); - size_t x_post_ffn = gb->add(x_post_ca, mlp_out); - std::string prefix = "model.decoder.layers." + std::to_string(layer_idx) + "."; - return x_post_ffn; -} -size_t MoonshineModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) -{ - if (!audio_features.empty()) { - float min_val = audio_features[0]; - float max_val = audio_features[0]; - double sum_val = 0.0; - for (float val : audio_features) { - if (val < min_val) min_val = val; - if (val > max_val) max_val = val; - sum_val += val; - } - (void)sum_val; - } - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.size() != 3) { - input_shape = {1, 1, static_cast(audio_features.size())}; - } - if (input_shape[1] != 1) { - throw std::runtime_error("Moonshine NPU encoder expects mono audio input"); - } - size_t expected_samples = static_cast(input_shape[2]); - std::vector<__fp16> audio_f16(expected_samples, 0); - size_t copy_samples = std::min(audio_features.size(), expected_samples); - if (copy_samples > 0) { - cactus_fp32_to_fp16(audio_features.data(), audio_f16.data(), copy_samples); - } - size_t T_enc = moonshine_downsampled_len(expected_samples); - size_t D_enc = static_cast(config_.hidden_dim); - if (T_enc == 0 || D_enc == 0) { - std::cout << "NPU encoder output has unexpected shape for input size " - << expected_samples << std::endl; - std::cout << "Falling back to CPU encoder path." << std::endl; - goto encoder_cpu_fallback; - } - - size_t total_elements = T_enc * D_enc; - std::vector<__fp16> npu_output(total_elements); - size_t elements_written = npu_encoder_->encode( - audio_f16.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - if (elements_written > 0) { - size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); - size_t enc_output_persistent = gb->persistent(enc_output_node); - last_encoder_post_norm_node_ = enc_output_persistent; - return enc_output_persistent; - } - } -encoder_cpu_fallback: - auto backend = - (config_.default_backend == Config::Backend::CPU) - ? ComputeBackend::CPU - : ComputeBackend::NPU; - size_t audio_input = 0; - const std::vector* cpu_audio = &audio_features; - std::vector<__fp16> audio_f16(cpu_audio->size()); - if (!cpu_audio->empty()) { - cactus_fp32_to_fp16(cpu_audio->data(), audio_f16.data(), cpu_audio->size()); - } - size_t audio_length = cpu_audio->size(); - audio_input = gb->input({1, 1, audio_length}, Precision::FP16); - gb->set_input(audio_input, audio_f16.data(), Precision::FP16); - size_t conv2_transposed = build_audio_preprocessor(gb, audio_input); - const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; - if (conv_shape.size() != 3 || conv_shape[0] != 1) - throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); - size_t T_enc = conv_shape[1]; - size_t D_enc = conv_shape[2]; - size_t h = gb->reshape(conv2_transposed, {T_enc, D_enc}); - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i){ - h = build_encoder_transformer_block(gb, h, i, backend, false, 0); - } - size_t h_norm = gb->layernorm( - h, - weight_nodes_.encoder_layer_norm_weight - ); - size_t h_norm_persistent = gb->persistent(h_norm); - last_encoder_post_norm_node_ = h_norm_persistent; - return h_norm_persistent; -} -size_t MoonshineModel::build_decoder(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - const size_t full_len = tokens.size(); - if (full_len == 0) { - throw std::runtime_error("Decoder token list cannot be empty."); - } - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; - size_t new_tokens = full_len - start_idx; - size_t position_offset = use_cache ? kv_cache_.current_seq_len : 0; - size_t tok_input = gb->input({new_tokens}, Precision::FP32); - std::vector tok_f(new_tokens); - for (size_t i = 0; i < new_tokens; i++) { - tok_f[i] = static_cast(tokens[start_idx + i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); - for (uint32_t layer_idx = 0; layer_idx < config_.num_decoder_layers; ++layer_idx) { - dec_hidden = build_decoder_transformer_block( - gb, - dec_hidden, - layer_idx, - backend, - use_cache, - position_offset - ); - } - size_t dec_norm = gb->layernorm( - dec_hidden, - weight_nodes_.decoder_norm_weight - ); - size_t logits_input = dec_norm; - if (last_token_only) { - size_t row_index = new_tokens - 1; - logits_input = gb->slice(logits_input, 0, row_index, 1); - } - auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; - size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); - last_new_tokens_ = new_tokens; - return logits; -} -size_t MoonshineModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) -{ - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - build_encoder(gb, audio_features); - size_t logits = build_decoder(tokens, use_cache, true); - return logits; -} -std::vector MoonshineModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - build_encoder(gb, audio_features); - size_t pooled = gb->mean(last_encoder_post_norm_node_, 0); - gb->execute(); - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - reset_cache(); - return embedding; -} -uint32_t MoonshineModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (audio_features.empty()) - throw std::runtime_error("Audio features cannot be empty in Moonshine decode_with_audio"); - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - bool cold_start = !encoder_ready_; - size_t logits_node = 0; - uint32_t bos = static_cast(get_tokenizer()->get_bos_token()); - std::vector full_tokens; - full_tokens.reserve(tokens.size() + 1); - full_tokens.push_back(bos); - full_tokens.insert(full_tokens.end(), tokens.begin(), tokens.end()); - if (cold_start) - { - gb->soft_reset(); - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - encoder_kv_ready_ = false; - first_decode_step_ = true; - build_encoder(gb, audio_features); - logits_node = build_decoder(full_tokens, false, false); - encoder_ready_ = true; - } - else - { - gb->soft_reset(); - reset_graph_side_cache_nodes(); - std::vector last_token_vec = { tokens.back() }; - logits_node = build_decoder(last_token_vec, true, true); - } - size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty); - if (!profile_file.empty()) gb->execute(profile_file); - else gb->execute(); - compute_entropy(gb, logits_node, out_entropy); - post_execute_updates(gb, full_tokens.size()); - update_kv_cache(gb, last_new_tokens_); - auto* out_ptr = gb->get_output(sampled_token_id); - uint32_t sampled = *reinterpret_cast(out_ptr); - record_sampled_token(sampled); - return sampled; -} -} -} diff --git a/cactus/models/model_needle.cpp b/cactus/models/model_needle.cpp deleted file mode 100644 index e7220a97f..000000000 --- a/cactus/models/model_needle.cpp +++ /dev/null @@ -1,337 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include - -namespace cactus { -namespace engine { - -namespace { - -size_t delta_rms_norm(CactusGraph* gb, size_t input, size_t weight, float epsilon) { - return gb->rms_norm(input, gb->scalar_add(weight, 1.0f), epsilon); -} - -size_t normalize_qk_proj(CactusGraph* gb, size_t proj, size_t norm_weight, - size_t seq_len, size_t num_heads, size_t head_dim, float epsilon) { - proj = gb->reshape(proj, {seq_len * num_heads, head_dim}); - proj = delta_rms_norm(gb, proj, norm_weight, epsilon); - return gb->reshape(proj, {seq_len, num_heads * head_dim}); -} - -size_t apply_residual_gate(CactusGraph* gb, size_t residual, size_t update, size_t gate_weight) { - return gb->add_clipped(residual, gb->multiply(update, gb->sigmoid(gate_weight))); -} - -size_t embed_tokens(CactusGraph* gb, size_t embedding_id, const std::vector& tokens, size_t hidden_dim) { - size_t n = tokens.size(); - size_t input_node = gb->input({n}, Precision::FP32); - std::vector data(n); - for (size_t i = 0; i < n; ++i) data[i] = static_cast(tokens[i]); - gb->set_input(input_node, data.data(), Precision::FP32); - return gb->scalar_multiply(gb->embedding(embedding_id, input_node), std::sqrt(static_cast(hidden_dim))); -} - -} // namespace - -NeedleModel::NeedleModel() : Model() {} - -NeedleModel::NeedleModel(const Config& config) : Model(config) { - weight_nodes_.encoder_layers.resize(config.num_encoder_layers); - weight_nodes_.decoder_layers.resize(config.num_decoder_layers); - float hd = static_cast(config.attention_head_dim); - attention_scale_ = 1.0f / std::sqrt(hd > 0.0f ? hd : 64.0f); - encoder_k_persistent_.assign(config.num_decoder_layers, 0); - encoder_v_persistent_.assign(config.num_decoder_layers, 0); -} - -void NeedleModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_layer_norm_weight.weights"); - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) { - auto& l = weight_nodes_.encoder_layers[i]; - std::string p = model_folder_path_ + "/encoder_layer_" + std::to_string(i) + "_"; - l.attn_gate_weight = gb->mmap_weights(p + "attn_gate.weights"); - l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); - l.attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); - l.attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); - l.attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); - l.attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); - l.attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); - l.attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); - } - - for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) { - auto& l = weight_nodes_.decoder_layers[i]; - std::string p = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - l.self_attn_gate_weight = gb->mmap_weights(p + "self_attn_gate.weights"); - l.cross_attn_gate_weight = gb->mmap_weights(p + "cross_attn_gate.weights"); - l.input_norm_weight = gb->mmap_weights(p + "input_norm.weights"); - l.post_attn_norm_weight = gb->mmap_weights(p + "post_attn_norm.weights"); - l.self_attn_q_weight = gb->mmap_weights(p + "attn_q.weights"); - l.self_attn_k_weight = gb->mmap_weights(p + "attn_k.weights"); - l.self_attn_v_weight = gb->mmap_weights(p + "attn_v.weights"); - l.self_attn_output_weight = gb->mmap_weights(p + "attn_output.weights"); - l.self_attn_q_norm_weight = gb->mmap_weights(p + "attn_q_norm.weights"); - l.self_attn_k_norm_weight = gb->mmap_weights(p + "attn_k_norm.weights"); - l.encoder_attn_q_weight = gb->mmap_weights(p + "encoder_attn_q.weights"); - l.encoder_attn_k_weight = gb->mmap_weights(p + "encoder_attn_k.weights"); - l.encoder_attn_v_weight = gb->mmap_weights(p + "encoder_attn_v.weights"); - l.encoder_attn_output_weight = gb->mmap_weights(p + "encoder_attn_output.weights"); - l.encoder_attn_q_norm_weight = gb->mmap_weights(p + "encoder_attn_q_norm.weights"); - l.encoder_attn_k_norm_weight = gb->mmap_weights(p + "encoder_attn_k_norm.weights"); - } -} - -void NeedleModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(config_.num_decoder_layers, 0); - cache_v_output_nodes_.assign(config_.num_decoder_layers, 0); -} - -void NeedleModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - auto* gb = static_cast(graph_handle_); - if (!gb) { reset_graph_side_cache_nodes(); return; } - - if (last_encoder_post_norm_node_ != 0) { - gb->invalidate_persistent(last_encoder_post_norm_node_); - last_encoder_post_norm_node_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { gb->invalidate_persistent(encoder_k_persistent_[i]); encoder_k_persistent_[i] = 0; } - if (encoder_v_persistent_[i] != 0) { gb->invalidate_persistent(encoder_v_persistent_[i]); encoder_v_persistent_[i] = 0; } - } - reset_graph_side_cache_nodes(); -} - -size_t NeedleModel::build_encoder_self_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool /*use_cache*/, size_t /*position_offset*/) { - const auto& l = weight_nodes_.encoder_layers[layer_idx]; - auto q = gb->matmul(input, l.attn_q_weight, true, backend); - auto k = gb->matmul(input, l.attn_k_weight, true, backend); - auto v = gb->matmul(input, l.attn_v_weight, true, backend); - - size_t seq = gb->get_output_buffer(q).shape[0]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - q = normalize_qk_proj(gb, q, l.attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); - k = normalize_qk_proj(gb, k, l.attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); - - auto q4 = gb->reshape(q, {1, seq, nh, hd}); - auto k4 = gb->reshape(k, {1, seq, nkv, hd}); - auto v4 = gb->reshape(v, {1, seq, nkv, hd}); - if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, 0); k4 = gb->rope(k4, config_.rope_theta, 0); } - - auto attn = gb->attention(q4, k4, v4, attention_scale_, false); - return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.attn_output_weight, true, backend); -} - -size_t NeedleModel::build_decoder_self_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - auto q = gb->matmul(input, l.self_attn_q_weight, true, backend); - auto k = gb->matmul(input, l.self_attn_k_weight, true, backend); - auto v = gb->matmul(input, l.self_attn_v_weight, true, backend); - - size_t seq = gb->get_output_buffer(q).shape[0]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - q = normalize_qk_proj(gb, q, l.self_attn_q_norm_weight, seq, nh, hd, config_.layer_norm_eps); - k = normalize_qk_proj(gb, k, l.self_attn_k_norm_weight, seq, nkv, hd, config_.layer_norm_eps); - - auto q4 = gb->reshape(q, {1, seq, nh, hd}); - auto k4 = gb->reshape(k, {1, seq, nkv, hd}); - auto v4 = gb->reshape(v, {1, seq, nkv, hd}); - if (config_.rope_theta > 0) { q4 = gb->rope(q4, config_.rope_theta, position_offset); k4 = gb->rope(k4, config_.rope_theta, position_offset); } - - size_t final_k = k4, final_v = v4; - if (use_cache && !kv_cache_.is_empty()) { - size_t cache_len = kv_cache_.current_seq_len; - size_t ck = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); - size_t cv = gb->input({1, cache_len, nkv, hd}, kv_cache_.precision); - auto kv = kv_cache_.get_key_view(layer_idx); - auto vv = kv_cache_.get_value_view(layer_idx); - if (!kv.ptr2 && !vv.ptr2) { - gb->set_external_input(ck, const_cast(kv.ptr1), kv_cache_.precision); - gb->set_external_input(cv, const_cast(vv.ptr1), kv_cache_.precision); - } else { - gb->set_external_input(ck, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cv, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - final_k = gb->concat(ck, k4, 1); - final_v = gb->concat(cv, v4, 1); - } - - cache_k_output_nodes_[layer_idx] = use_cache ? final_k : k4; - cache_v_output_nodes_[layer_idx] = use_cache ? final_v : v4; - - auto attn = gb->attention(q4, final_k, final_v, attention_scale_, position_offset); - return gb->matmul(gb->reshape(attn, {seq, nh * hd}), l.self_attn_output_weight, true, backend); -} - -size_t NeedleModel::build_decoder_cross_attention(CactusGraph* gb, size_t input, - uint32_t layer_idx, ComputeBackend backend, - bool /*use_cache*/, size_t /*position_offset*/) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - size_t nh = config_.attention_heads, nkv = config_.attention_kv_heads, hd = config_.attention_head_dim; - - auto q = gb->matmul(input, l.encoder_attn_q_weight, true, backend); - size_t seq_dec = gb->get_output_buffer(q).shape[0]; - q = normalize_qk_proj(gb, q, l.encoder_attn_q_norm_weight, seq_dec, nh, hd, config_.layer_norm_eps); - auto q4 = gb->reshape(q, {1, seq_dec, nh, hd}); - - size_t k4 = 0, v4 = 0; - bool cached = encoder_k_persistent_[layer_idx] != 0 && gb->is_populated(encoder_k_persistent_[layer_idx]); - if (cached) { - k4 = encoder_k_persistent_[layer_idx]; - v4 = encoder_v_persistent_[layer_idx]; - } else { - auto kp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_k_weight, true, backend); - auto vp = gb->matmul(last_encoder_post_norm_node_, l.encoder_attn_v_weight, true, backend); - size_t seq_enc = gb->get_output_buffer(kp).shape[0]; - kp = normalize_qk_proj(gb, kp, l.encoder_attn_k_norm_weight, seq_enc, nkv, hd, config_.layer_norm_eps); - k4 = gb->reshape(kp, {1, seq_enc, nkv, hd}); - v4 = gb->reshape(vp, {1, seq_enc, nkv, hd}); - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k4); - encoder_v_persistent_[layer_idx] = gb->persistent(v4); - } - k4 = encoder_k_persistent_[layer_idx]; - v4 = encoder_v_persistent_[layer_idx]; - } - - auto attn = gb->attention(q4, k4, v4, attention_scale_, false); - return gb->matmul(gb->reshape(attn, {seq_dec, nh * hd}), l.encoder_attn_output_weight, true, backend); -} - -size_t NeedleModel::build_encoder_transformer_block(CactusGraph* gb, size_t hidden, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.encoder_layers[layer_idx]; - auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); - auto attn = build_encoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - return apply_residual_gate(gb, hidden, attn, l.attn_gate_weight); -} - -size_t NeedleModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, - uint32_t layer_idx, ComputeBackend backend, - bool use_cache, size_t position_offset) { - const auto& l = weight_nodes_.decoder_layers[layer_idx]; - auto normed = delta_rms_norm(gb, hidden, l.input_norm_weight, config_.layer_norm_eps); - auto self_out = build_decoder_self_attention(gb, normed, layer_idx, backend, use_cache, position_offset); - auto after_self = apply_residual_gate(gb, hidden, self_out, l.self_attn_gate_weight); - auto cross_in = delta_rms_norm(gb, after_self, l.post_attn_norm_weight, config_.layer_norm_eps); - auto cross_out = build_decoder_cross_attention(gb, cross_in, layer_idx, backend, use_cache, position_offset); - return apply_residual_gate(gb, after_self, cross_out, l.cross_attn_gate_weight); -} - -void NeedleModel::run_encoder(const std::vector& encoder_tokens) { - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - auto hidden = embed_tokens(gb, embedding_node_id_, encoder_tokens, config_.hidden_dim); - - for (uint32_t i = 0; i < config_.num_encoder_layers; ++i) - hidden = build_encoder_transformer_block(gb, hidden, i, backend, false, 0); - - last_encoder_post_norm_node_ = gb->persistent( - delta_rms_norm(gb, hidden, weight_nodes_.encoder_norm_weight, config_.layer_norm_eps)); -} - -size_t NeedleModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - size_t pos_offset = use_cache ? kv_cache_.current_seq_len : 0; - - auto hidden = embed_tokens(gb, embedding_node_id_, tokens, config_.hidden_dim); - for (uint32_t i = 0; i < config_.num_decoder_layers; ++i) - hidden = build_decoder_transformer_block(gb, hidden, i, backend, use_cache, pos_offset); - - auto norm = delta_rms_norm(gb, hidden, weight_nodes_.decoder_norm_weight, config_.layer_norm_eps); - if (last_token_only) norm = gb->slice(norm, 0, tokens.size() - 1, 1); - return gb->matmul(norm, output_weight_node_id_, true, backend); -} - -size_t NeedleModel::forward(const std::vector& tokens, bool use_cache) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - if (!encoder_ready_ || !use_cache) { - std::vector enc(tokens.begin(), tokens.end() - 1); - std::vector seed(tokens.end() - 1, tokens.end()); - if (!use_cache) { reset_cache(); gb->soft_reset(); } - run_encoder(enc); - encoder_ready_ = true; - return run_decoder_step(seed, false, false); - } - return run_decoder_step(tokens, true, tokens.size() == 1); -} - -void NeedleModel::prefill(const std::vector& tokens, size_t /*chunk_size*/, const std::string& profile_file) { - if (tokens.empty()) return; - reset_cache(); - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - run_encoder(tokens); - if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); - encoder_ready_ = true; -} - -uint32_t NeedleModel::decode(const std::vector& tokens, - float temperature, float top_p, size_t top_k, - const std::string& profile_file, float* out_entropy, - float /*min_p*/, float /*repetition_penalty*/) { - if (temperature < 0) temperature = config_.default_temperature; - if (top_p < 0) top_p = config_.default_top_p; - if (top_k == 0) top_k = config_.default_top_k; - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - bool cold_start = kv_cache_.is_empty(); - std::vector cold_tokens; - const std::vector* dec_tokens = &tokens; - if (!encoder_ready_) { - reset_cache(); gb->soft_reset(); - std::vector enc(tokens.begin(), tokens.end() - 1); - cold_tokens.assign(tokens.end() - 1, tokens.end()); - run_encoder(enc); - encoder_ready_ = true; - dec_tokens = &cold_tokens; - cold_start = true; - } - - size_t logits = cold_start - ? run_decoder_step(*dec_tokens, false, false) - : run_decoder_step(tokens, true, tokens.size() == 1); - - if (config_.final_logit_softcapping > 0.0f) { - float inv = 1.0f / config_.final_logit_softcapping; - logits = gb->scalar_multiply(gb->tanh(gb->scalar_multiply(logits, inv)), config_.final_logit_softcapping); - } - - auto sampled = sample_token(gb, logits, temperature, top_p, top_k, 0.0f, 1.0f); - if (!profile_file.empty()) gb->execute(profile_file); else gb->execute(); - - compute_entropy(gb, logits, out_entropy); - post_execute_updates(gb, tokens.size()); - update_kv_cache(gb, dec_tokens->size()); - return *static_cast(gb->get_output(sampled)); -} - -} // namespace engine -} // namespace cactus diff --git a/cactus/models/model_nomic.cpp b/cactus/models/model_nomic.cpp deleted file mode 100644 index a9477a865..000000000 --- a/cactus/models/model_nomic.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include - - -namespace cactus { -namespace engine { - -NomicModel::NomicModel() : Model() {} - -NomicModel::NomicModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void NomicModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.embedding_layernorm_weight = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.weight"); - weight_nodes_.embedding_layernorm_bias = gb->mmap_weights(model_folder_path_ + "/embedding_layernorm.bias"); - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_q_bias = gb->mmap_weights(layer_prefix + "attn_q.bias"); - layer.attn_k_bias = gb->mmap_weights(layer_prefix + "attn_k.bias"); - layer.attn_v_bias = gb->mmap_weights(layer_prefix + "attn_v.bias"); - - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output.bias"); - layer.ffn_norm_1_weight = gb->mmap_weights(layer_prefix + "norm1.weights"); - layer.ffn_norm_1_bias = gb->mmap_weights(layer_prefix + "norm1.bias"); - layer.ffn_norm_2_weight = gb->mmap_weights(layer_prefix + "norm2.weights"); - layer.ffn_norm_2_bias = gb->mmap_weights(layer_prefix + "norm2.bias"); - - if ((i + 1) % config_.moe_every_n_layers != 0) { - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.ffn_up_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.ffn_down_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - } else { - layer.mlp_router_layer_weight = gb->mmap_weights(layer_prefix + "mlp_router.layer.weights"); - layer.mlp_experts_bias = gb->mmap_weights(layer_prefix + "mlp_experts.bias"); - - for (uint32_t j = 0; j < config_.num_experts; j++) { - layer.mlp_experts_mlp1_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp1.weights")); - layer.mlp_experts_mlp2_weight.push_back(gb->mmap_weights(layer_prefix + "mlp_expert_" + std::to_string(j) + ".mlp2.weights")); - } - } - } -} - -size_t NomicModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - (void)position_offset; - - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - q_proj = gb->add(q_proj, layer.attn_q_bias); - - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - k_proj = gb->add(k_proj, layer.attn_k_bias); - - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - v_proj = gb->add(v_proj, layer.attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - const size_t seq_len = q_shape[0]; - const size_t num_heads = config_.attention_heads; - const size_t head_dim = config_.attention_head_dim; - - if (num_heads == 0 || head_dim == 0) { - throw std::runtime_error("Invalid attention head configuration for Nomic model"); - } - - auto reshape_to_heads = [&](size_t tensor) { - return gb->reshape(tensor, {1, seq_len, num_heads, head_dim}); - }; - - auto q_proj_4d = reshape_to_heads(q_proj); - auto k_proj_4d = reshape_to_heads(k_proj); - auto v_proj_4d = reshape_to_heads(v_proj); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, 0); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, 0); - } - - auto attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, false); - auto attn_output = gb->reshape(attn_output_4d, {seq_len, num_heads * head_dim}); - - auto output = gb->matmul(attn_output, layer.attn_output_weight, true, backend); - output = gb->add(output, layer.attn_output_bias); - return output; -} - -size_t NomicModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - if ((layer_idx + 1) % config_.moe_every_n_layers != 0) { - return build_standard_mlp(gb, normalized_h, layer_idx, backend); - } else { - return build_moe_mlp(gb, normalized_h, layer_idx, backend); - } -} - -size_t NomicModel::build_standard_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto hidden = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - hidden = gb->add(hidden, layer.ffn_up_bias); - hidden = gb->gelu(hidden); - hidden = gb->matmul(hidden, layer.ffn_down_weight, true, backend); - hidden = gb->add(hidden, layer.ffn_down_bias); - return hidden; -} - -size_t NomicModel::build_moe_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - const size_t num_experts = config_.num_experts != 0 - ? config_.num_experts - : gb->get_output_buffer(layer.mlp_router_layer_weight).shape[0]; - - auto gate_weights = gb->matmul(normalized_h, layer.mlp_router_layer_weight, true, backend); - auto gate_probs = gb->softmax(gate_weights); - auto topk_result = gb->topk(gate_probs, config_.num_top_experts); - auto topk_idx = gb->index(topk_result, 0, 0); - - std::vector w1_weights, w2_weights; - for (size_t e = 0; e < num_experts; ++e) { - w1_weights.push_back(layer.mlp_experts_mlp1_weight[e]); - w2_weights.push_back(layer.mlp_experts_mlp2_weight[e]); - } - - auto moe_out = gb->moe_layer( - normalized_h, gate_probs, topk_idx, - w1_weights, w2_weights, - num_experts, config_.num_top_experts, - false, - 1e-6f, - 1.0f, - Activation::GELU - ); - - return gb->add(moe_out, layer.mlp_experts_bias); -} - -size_t NomicModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - (void)position_offset; - const auto& layer = weight_nodes_.layers[layer_idx]; - auto attn_output = build_attention(gb, hidden, layer_idx, backend); - auto residual = gb->add(hidden, attn_output); - auto normalized_residual = gb->layernorm(residual, layer.ffn_norm_1_weight, layer.ffn_norm_1_bias, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_residual, layer_idx, backend); - auto final_residual = gb->add(normalized_residual, mlp_output); - auto normalized_final_residual = gb->layernorm(final_residual, layer.ffn_norm_2_weight, layer.ffn_norm_2_bias, config_.layer_norm_eps); - return normalized_final_residual; -} - -size_t NomicModel::forward(const std::vector& tokens, bool use_cache) { - if (use_cache) { - throw std::runtime_error("NomicModel does not support generation, it's an encoder model"); - } - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t seq_len = static_cast(tokens.size()); - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t input_node_id = gb->input({seq_len}, Precision::FP32); - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - size_t hidden = gb->embedding(embedding_node_id_, input_node_id); - - hidden = gb->layernorm(hidden, weight_nodes_.embedding_layernorm_weight, weight_nodes_.embedding_layernorm_bias, config_.layer_norm_eps); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend); - } - - return hidden; -} - -} -} diff --git a/cactus/models/model_parakeet.cpp b/cactus/models/model_parakeet.cpp deleted file mode 100644 index 7aee878b3..000000000 --- a/cactus/models/model_parakeet.cpp +++ /dev/null @@ -1,860 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include -#include - -size_t shape_elements(const std::vector& shape) { - if (shape.empty()) return 0; - size_t total = 1; - for (int d : shape) { - if (d <= 0) return 0; - total *= static_cast(d); - } - return total; -} - -bool pack_parakeet_features_for_npu( - const std::vector<__fp16>& time_major_f16, - size_t frames, - size_t num_mels, - const std::vector& input_shape, - std::vector<__fp16>& packed) -{ - if (input_shape.empty()) return false; - const size_t total = shape_elements(input_shape); - if (total == 0) return false; - packed.assign(total, static_cast<__fp16>(0.0f)); - - auto tm = [&](size_t t, size_t m) -> __fp16 { - return time_major_f16[t * num_mels + m]; - }; - - if (input_shape.size() == 4) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - const size_t s3 = static_cast(input_shape[3]); - if (s0 != 1) return false; - - if (s1 == 1 && s2 >= frames && s3 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(t * s3) + m] = tm(t, m); - } - } - return true; - } - if (s1 >= frames && s2 == num_mels && s3 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[((t * s2 + m) * s3)] = tm(t, m); - } - } - return true; - } - if (s1 == num_mels && s2 >= frames && s3 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[((m * s2 + t) * s3)] = tm(t, m); - } - } - return true; - } - if (s1 == 1 && s2 == num_mels && s3 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(m * s3) + t] = tm(t, m); - } - } - return true; - } - return false; - } - - if (input_shape.size() == 3) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - const size_t s2 = static_cast(input_shape[2]); - - if (s0 == 1 && s1 >= frames && s2 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[t * s2 + m] = tm(t, m); - } - } - return true; - } - if (s0 == 1 && s1 == num_mels && s2 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[m * s2 + t] = tm(t, m); - } - } - return true; - } - if (s0 >= frames && s1 == num_mels && s2 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(t * s1 + m) * s2] = tm(t, m); - } - } - return true; - } - if (s0 == num_mels && s1 >= frames && s2 == 1) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[(m * s1 + t) * s2] = tm(t, m); - } - } - return true; - } - return false; - } - - if (input_shape.size() == 2) { - const size_t s0 = static_cast(input_shape[0]); - const size_t s1 = static_cast(input_shape[1]); - - if (s0 >= frames && s1 == num_mels) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[t * s1 + m] = tm(t, m); - } - } - return true; - } - if (s0 == num_mels && s1 >= frames) { - for (size_t t = 0; t < frames; ++t) { - for (size_t m = 0; m < num_mels; ++m) { - packed[m * s1 + t] = tm(t, m); - } - } - return true; - } - return false; - } - - return false; -} - -bool infer_npu_encoder_output_shape( - const std::vector& output_shape, - size_t elements_written, - size_t fallback_hidden_dim, - size_t& time_steps, - size_t& hidden_dim) -{ - if (elements_written == 0) return false; - - std::vector dims; - dims.reserve(output_shape.size()); - for (int d : output_shape) { - if (d > 0) dims.push_back(static_cast(d)); - } - - if (dims.size() >= 2) { - time_steps = dims[dims.size() - 2]; - hidden_dim = dims[dims.size() - 1]; - } else if (dims.size() == 1) { - hidden_dim = dims[0]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } else { - hidden_dim = fallback_hidden_dim; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } - - if (time_steps == 0 || hidden_dim == 0) return false; - if (time_steps * hidden_dim > elements_written) return false; - return true; -} - -namespace cactus { -namespace engine { - -ParakeetModel::ParakeetModel() : Model() {} - -ParakeetModel::ParakeetModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - - // Legacy converted Parakeet configs inherited generic rope_theta defaults. - // Keep RoPE disabled unless explicitly set to a non-default value. - if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || - std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { - config_.rope_theta = 0.0f; - } -} - -void ParakeetModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - - weight_nodes_.ctc_head_weight = gb->mmap_weights(model_folder_path_ + "/ctc_head_weight.weights"); - weight_nodes_.ctc_head_bias = gb->mmap_weights(model_folder_path_ + "/ctc_head_bias.bias"); - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const std::filesystem::path model_path(model_folder_path_); - has_cpu_encoder_weights_ = - std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && - std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); - - if (has_cpu_encoder_weights_) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - } - - for (uint32_t i = 0; i < config_.num_layers; ++i) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); - layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); - layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); - layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); - - layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); - layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); - layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); - layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); - - layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); - layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); - layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); - layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); - - layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); - layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); - layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); - layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); - layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); - layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); - layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); - layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); - layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); - layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); - - layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); - layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); - layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); - layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); - layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); - layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); - layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); - layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); - } -} - -size_t ParakeetModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> features_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); - - size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); - gb->set_input(x, features_f16.data(), Precision::FP16); - - x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); - x = gb->relu(x); - - const auto& conv_shape = gb->get_output_buffer(x).shape; - if (conv_shape.size() != 4 || conv_shape[0] != 1) { - throw std::runtime_error("Parakeet subsampling produced invalid shape"); - } - - const size_t C = conv_shape[1]; - const size_t T = conv_shape[2]; - const size_t W = conv_shape[3]; - - size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); - size_t flattened = gb->reshape(t_major, {T, C * W}); - size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); - projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); - return projected; -} - -size_t ParakeetModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { - const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - const size_t half_dim = hidden_dim / 2; - const size_t rel_len = 2 * seq_len - 1; - - std::vector pos_embed(rel_len * hidden_dim, 0.0f); - for (size_t p = 0; p < rel_len; ++p) { - const int rel_pos = static_cast(seq_len - 1) - static_cast(p); - for (size_t i = 0; i < half_dim; ++i) { - const float exponent = static_cast(2 * i) / static_cast(hidden_dim); - const float inv_freq = 1.0f / std::pow(10000.0f, exponent); - const float angle = static_cast(rel_pos) * inv_freq; - pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); - if (2 * i + 1 < hidden_dim) { - pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); - } - } - } - - std::vector<__fp16> pos_embed_f16(pos_embed.size()); - cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); - - size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); - gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); - return pos_node; -} - -size_t ParakeetModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); - q = gb->add(q, layer.self_attn_q_bias); - size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); - k = gb->add(k, layer.self_attn_k_bias); - size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); - v = gb->add(v, layer.self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("Parakeet self-attention expects [T, D]"); - } - - const size_t T = q_shape[0]; - const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); - const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); - const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); - - size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); - size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); - - size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); - size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); - size_t q_u4 = gb->add(q4, bias_u); - size_t q_v4 = gb->add(q4, bias_v); - - size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); - size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); - - size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); - attn = gb->reshape(attn, {T, q_heads * head_dim}); - - size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); - out = gb->add(out, layer.self_attn_output_bias); - return out; -} - -size_t ParakeetModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, bool second_ff, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; - size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; - size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; - size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; - - size_t x = gb->matmul(hidden, w1, true, backend); - x = gb->add(x, b1); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), ::tolower); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->matmul(x, w2, true, backend); - x = gb->add(x, b2); - return x; -} - -size_t ParakeetModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { - (void)backend; - const auto& layer = weight_nodes_.layers[layer_idx]; - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet convolution module expects [T, D]"); - } - - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t x = gb->reshape(hidden, {1, T, D}); - x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); - x = gb->glu(x, -1); - - x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); - - x = gb->batchnorm( - x, - layer.conv_batchnorm_weight, - layer.conv_batchnorm_bias, - layer.conv_batchnorm_running_mean, - layer.conv_batchnorm_running_var, - 2, - 1e-5f - ); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), ::tolower); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); - - x = gb->reshape(x, {T, D}); - return x; -} - -size_t ParakeetModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); - size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); - ff1 = gb->scalar_multiply(ff1, 0.5f); - size_t x = gb->add(hidden, ff1); - - size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); - size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); - x = gb->add(x, attn); - - size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); - size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); - x = gb->add(x, conv); - - size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); - size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); - ff2 = gb->scalar_multiply(ff2, 0.5f); - x = gb->add(x, ff2); - - x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); - return x; -} - -size_t ParakeetModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("Parakeet expects audio_features with shape [num_mels, num_frames]"); - } - const size_t frames = audio_features.size() / num_mels; - size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t expected_vocab_size = std::max(1, static_cast(config_.vocab_size)); - { - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - expected_vocab_size = ctc_w_shape[0]; - expected_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - expected_vocab_size = ctc_w_shape[0]; - expected_hidden_dim = ctc_w_shape[1]; - } - } - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> time_major_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); - - std::vector input_shape = npu_encoder_->get_input_shape(); - std::vector<__fp16> npu_input; - if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { - npu_encoder_->preallocate(input_shape, "x", ""); - - std::vector output_shape = npu_encoder_->get_output_shape(); - size_t output_capacity = npu_encoder_->get_output_buffer_size(); - if (output_capacity == 0) { - output_capacity = shape_elements(output_shape); - } - if (output_capacity == 0) { - output_capacity = std::max( - 1, frames * std::max(1, static_cast(config_.hidden_dim))); - } - - std::vector<__fp16> npu_output(output_capacity); - const size_t elements_written = npu_encoder_->encode( - npu_input.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - size_t T_enc = 0; - size_t D_enc = 0; - if (elements_written > 0 && - infer_npu_encoder_output_shape(output_shape, elements_written, - static_cast(config_.hidden_dim), T_enc, D_enc)) { - const __fp16* src = npu_output.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - const size_t cached_count = npu_encoder_->get_output_buffer_size(); - const size_t required = T_enc * D_enc; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - // Accept encoder hidden output [T, hidden] or direct logits [T, vocab]. - if (D_enc == expected_hidden_dim || D_enc == expected_vocab_size) { - size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output, src, Precision::FP16); - return enc_output; - } - } - } - } - - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet requires either CPU encoder weights or model.mlpackage encoder output."); - } - - ComputeBackend backend = ComputeBackend::CPU; - size_t hidden = build_subsampling(gb, audio_features); - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet encoder expects subsampling output [T, D]"); - } - size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); - for (uint32_t i = 0; i < config_.num_layers; ++i) { - hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); - } - return hidden; -} - -size_t ParakeetModel::build_ctc_logits(CactusGraph* gb, size_t hidden_states) { - const auto& hidden_shape = gb->get_output_buffer(hidden_states).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("Parakeet CTC head expects hidden states [T, D]"); - } - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t hidden_nlc = gb->reshape(hidden_states, {1, T, D}); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - size_t vocab_size = ctc_w_shape.empty() ? config_.vocab_size : ctc_w_shape[0]; - size_t logits_nlc = gb->conv1d_pointwise(hidden_nlc, weight_nodes_.ctc_head_weight, weight_nodes_.ctc_head_bias); - - return gb->reshape(logits_nlc, {T, vocab_size}); -} - -size_t ParakeetModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) { - (void)tokens; - (void)use_cache; - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - size_t encoder_out = build_encoder(gb, audio_features); - - const auto& enc_shape = gb->get_output_buffer(encoder_out).shape; - if (enc_shape.size() != 2) { - throw std::runtime_error("Parakeet encoder output must be rank-2 [T, D]"); - } - - size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } - - // If the NPU graph already emits [T, vocab] logits, skip CPU CTC head. - if (enc_shape[1] == ctc_vocab_size && ctc_vocab_size != ctc_hidden_dim) { - return encoder_out; - } - - if (enc_shape[1] != ctc_hidden_dim) { - throw std::runtime_error( - "Parakeet encoder output dim mismatch: expected hidden dim " + - std::to_string(ctc_hidden_dim) + ", got " + std::to_string(enc_shape[1])); - } - - return build_ctc_logits(gb, encoder_out); -} - -std::vector ParakeetModel::greedy_decode_tokens(CactusGraph* gb, size_t logits_node) const { - const auto& logits_buf = gb->get_output_buffer(logits_node); - if (logits_buf.shape.size() != 2) { - throw std::runtime_error("Parakeet logits must be rank-2 [T, vocab]"); - } - - const size_t T = logits_buf.shape[0]; - const size_t vocab_size = logits_buf.shape[1]; - std::vector frame_ids(T, 0); - - if (logits_buf.precision == Precision::FP32) { - const float* src = logits_buf.data_as(); - for (size_t t = 0; t < T; ++t) { - const float* row = src + t * vocab_size; - size_t best_idx = 0; - float best_val = row[0]; - for (size_t v = 1; v < vocab_size; ++v) { - if (row[v] > best_val) { - best_val = row[v]; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } else if (logits_buf.precision == Precision::FP16) { - const __fp16* src = logits_buf.data_as<__fp16>(); - for (size_t t = 0; t < T; ++t) { - const __fp16* row = src + t * vocab_size; - size_t best_idx = 0; - float best_val = static_cast(row[0]); - for (size_t v = 1; v < vocab_size; ++v) { - const float val = static_cast(row[v]); - if (val > best_val) { - best_val = val; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } else { - const int8_t* src = logits_buf.data_as(); - for (size_t t = 0; t < T; ++t) { - const int8_t* row = src + t * vocab_size; - size_t best_idx = 0; - int best_val = static_cast(row[0]); - for (size_t v = 1; v < vocab_size; ++v) { - const int val = static_cast(row[v]); - if (val > best_val) { - best_val = val; - best_idx = v; - } - } - frame_ids[t] = static_cast(best_idx); - } - } - - const uint32_t blank_id = config_.pad_token_id > 0 ? config_.pad_token_id : static_cast(vocab_size - 1); - std::vector decoded; - decoded.reserve(frame_ids.size()); - - uint32_t prev = blank_id; - for (uint32_t id : frame_ids) { - if (id == blank_id) { - prev = blank_id; - continue; - } - if (id == prev) { - continue; - } - decoded.push_back(id); - prev = id; - } - - return decoded; -} - -uint32_t ParakeetModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - (void)temperature; - (void)top_p; - (void)top_k; - (void)min_p; - (void)repetition_penalty; - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in Parakeet decode_with_audio"); - } - - auto* gb = static_cast(graph_handle_); - const bool new_request = !ctc_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; - if (new_request) { - gb->soft_reset(); - size_t logits_node = forward(audio_features, tokens, false); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - - ctc_tokens_ = greedy_decode_tokens(gb, logits_node); - ctc_emit_index_ = 0; - ctc_tokens_ready_ = true; - } - - last_input_token_count_ = tokens.size(); - if (out_entropy) { - *out_entropy = 0.0f; - } - - if (ctc_emit_index_ < ctc_tokens_.size()) { - return ctc_tokens_[ctc_emit_index_++]; - } - return get_tokenizer()->get_eos_token(); -} - -std::vector ParakeetModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - size_t hidden = build_encoder(gb, audio_features); - - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - size_t ctc_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - size_t ctc_vocab_size = std::max(1, static_cast(config_.vocab_size)); - const auto& ctc_w_shape = gb->get_output_buffer(weight_nodes_.ctc_head_weight).shape; - if (ctc_w_shape.size() == 2) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } else if (ctc_w_shape.size() == 3) { - ctc_vocab_size = ctc_w_shape[0]; - ctc_hidden_dim = ctc_w_shape[1]; - } - - // Embeddings require hidden states; if NPU returned logits, rerun CPU encoder. - if (hidden_shape.size() == 2 && - hidden_shape[1] == ctc_vocab_size && - ctc_vocab_size != ctc_hidden_dim && - use_npu_encoder_) { - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet audio embeddings require hidden-state encoder output; " - "CPU encoder fallback weights are not available."); - } - const bool prev_use_npu = use_npu_encoder_; - use_npu_encoder_ = false; - gb->soft_reset(); - hidden = build_encoder(gb, audio_features); - use_npu_encoder_ = prev_use_npu; - } - - size_t pooled = gb->mean(hidden, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - const size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - - if (output_buf.precision == Precision::FP32) { - const float* src = output_buf.data_as(); - std::copy(src, src + hidden_dim, embedding.begin()); - } else if (output_buf.precision == Precision::FP16) { - const __fp16* src = output_buf.data_as<__fp16>(); - Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); - } else { - const int8_t* src = output_buf.data_as(); - Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); - } - - reset_cache(); - return embedding; -} - -void ParakeetModel::reset_cache() { - Model::reset_cache(); - ctc_tokens_ready_ = false; - ctc_emit_index_ = 0; - ctc_tokens_.clear(); - last_input_token_count_ = 0; -} - -} // namespace engine -} // namespace cactus \ No newline at end of file diff --git a/cactus/models/model_parakeet_tdt.cpp b/cactus/models/model_parakeet_tdt.cpp deleted file mode 100644 index da0e827d4..000000000 --- a/cactus/models/model_parakeet_tdt.cpp +++ /dev/null @@ -1,1107 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include "../telemetry/telemetry.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -size_t shape_elements(const std::vector& shape); -bool pack_parakeet_features_for_npu( - const std::vector<__fp16>& time_major_f16, - size_t frames, - size_t num_mels, - const std::vector& input_shape, - std::vector<__fp16>& packed); - -namespace { - -constexpr uint32_t kMaxStreamDurationSkipFrames = 2; - -bool infer_npu_encoder_output_shape( - const std::vector& output_shape, - size_t elements_written, - size_t fallback_hidden_dim, - size_t& time_steps, - size_t& hidden_dim) -{ - if (elements_written == 0) return false; - - std::vector dims; - dims.reserve(output_shape.size()); - for (int d : output_shape) { - if (d > 0) dims.push_back(static_cast(d)); - } - - if (dims.size() >= 2) { - time_steps = dims[dims.size() - 2]; - hidden_dim = dims[dims.size() - 1]; - } else if (dims.size() == 1) { - hidden_dim = dims[0]; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } else { - hidden_dim = fallback_hidden_dim; - if (hidden_dim == 0 || (elements_written % hidden_dim) != 0) return false; - time_steps = elements_written / hidden_dim; - } - - if (time_steps == 0 || hidden_dim == 0) return false; - if (time_steps * hidden_dim > elements_written) return false; - return true; -} - -std::vector<__fp16> copy_buffer_to_fp16(const BufferDesc& buffer) { - std::vector<__fp16> out(buffer.total_size, static_cast<__fp16>(0.0f)); - if (buffer.total_size == 0) { - return out; - } - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>(); - std::copy(src, src + buffer.total_size, out.begin()); - return out; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as(); - cactus_fp32_to_fp16(src, out.data(), buffer.total_size); - return out; - } - - if (buffer.precision == Precision::INT8 && !buffer.is_interleaved) { - const int8_t* src = buffer.data_as(); - Quantization::int8_to_fp16(src, out.data(), buffer.total_size, 1.0f); - return out; - } - - throw std::runtime_error("Unsupported precision for FP16 conversion in ParakeetTDT"); -} - -size_t argmax_range(const BufferDesc& buffer, size_t offset, size_t length) { - if (length == 0 || (offset + length) > buffer.total_size) { - throw std::runtime_error("Invalid argmax range"); - } - - size_t best_idx = 0; - float best_val = -std::numeric_limits::infinity(); - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>() + offset; - best_val = static_cast(src[0]); - for (size_t i = 1; i < length; ++i) { - float v = static_cast(src[i]); - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as() + offset; - best_val = src[0]; - for (size_t i = 1; i < length; ++i) { - float v = src[i]; - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - if (buffer.precision == Precision::INT8) { - const int8_t* src = buffer.data_as() + offset; - best_val = static_cast(src[0]); - for (size_t i = 1; i < length; ++i) { - float v = static_cast(src[i]); - if (v > best_val) { - best_val = v; - best_idx = i; - } - } - return best_idx; - } - - throw std::runtime_error("Unsupported logits precision in argmax"); -} - -size_t argmax_range_with_bias(const BufferDesc& buffer, size_t offset, size_t length, - const std::unordered_map& bias) { - if (bias.empty()) { - return argmax_range(buffer, offset, length); - } - if (length == 0 || (offset + length) > buffer.total_size) { - throw std::runtime_error("Invalid argmax range"); - } - - size_t best_idx = 0; - float best_val = -std::numeric_limits::infinity(); - - if (buffer.precision == Precision::FP16) { - const __fp16* src = buffer.data_as<__fp16>() + offset; - for (size_t i = 0; i < length; ++i) { - float v = static_cast(src[i]); - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - if (buffer.precision == Precision::FP32) { - const float* src = buffer.data_as() + offset; - for (size_t i = 0; i < length; ++i) { - float v = src[i]; - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - if (buffer.precision == Precision::INT8) { - const int8_t* src = buffer.data_as() + offset; - for (size_t i = 0; i < length; ++i) { - float v = static_cast(src[i]); - auto it = bias.find(static_cast(i)); - if (it != bias.end()) v += it->second; - if (v > best_val) { best_val = v; best_idx = i; } - } - return best_idx; - } - - throw std::runtime_error("Unsupported logits precision in biased argmax"); -} - -} // namespace - -namespace cactus { -namespace engine { - -ParakeetTDTModel::ParakeetTDTModel() : Model() {} - -ParakeetTDTModel::ParakeetTDTModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - attention_scale_ = 1.0f / std::sqrt(hd); - - if (std::fabs(config_.rope_theta - 10000.0f) < 1e-3f || - std::fabs(config_.rope_theta - 1000000.0f) < 1e-3f) { - config_.rope_theta = 0.0f; - } -} - -void ParakeetTDTModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - - weight_nodes_.predictor_embed = gb->mmap_weights(model_folder_path_ + "/tdt_predictor_embed.weights"); - - size_t predictor_layers = static_cast(config_.predictor_num_layers); - if (predictor_layers == 0) { - while (true) { - const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(predictor_layers); - if (!std::filesystem::exists(prefix + "_weight_ih.weights") || - !std::filesystem::exists(prefix + "_weight_hh.weights") || - !std::filesystem::exists(prefix + "_bias.weights")) { - break; - } - ++predictor_layers; - } - } - if (predictor_layers == 0) { - predictor_layers = 1; - } - - weight_nodes_.predictor_layers.resize(predictor_layers); - for (size_t i = 0; i < predictor_layers; ++i) { - auto& layer = weight_nodes_.predictor_layers[i]; - const std::string prefix = model_folder_path_ + "/tdt_predictor_lstm_" + std::to_string(i); - layer.weight_ih = gb->mmap_weights(prefix + "_weight_ih.weights"); - layer.weight_hh = gb->mmap_weights(prefix + "_weight_hh.weights"); - layer.bias = gb->mmap_weights(prefix + "_bias.weights"); - } - - weight_nodes_.joint_enc_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.weights"); - weight_nodes_.joint_enc_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_enc.bias"); - weight_nodes_.joint_pred_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.weights"); - weight_nodes_.joint_pred_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_pred.bias"); - weight_nodes_.joint_out_weight = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.weights"); - weight_nodes_.joint_out_bias = gb->mmap_weights(model_folder_path_ + "/tdt_joint_out.bias"); - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - std::vector input_shape = npu_encoder_->get_input_shape(); - if (!input_shape.empty()) { - npu_encoder_->preallocate(input_shape, "x", ""); - } - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const std::filesystem::path model_path(model_folder_path_); - has_cpu_encoder_weights_ = - std::filesystem::exists(model_path / "subsampling_conv0_weight.weights") && - std::filesystem::exists(model_path / "subsampling_linear_weight.weights"); - - if (has_cpu_encoder_weights_) { - weight_nodes_.subsampling_conv0_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_weight.weights"); - weight_nodes_.subsampling_conv0_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_conv0_bias.bias"); - weight_nodes_.subsampling_depthwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_weight.weights"); - weight_nodes_.subsampling_depthwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise1_bias.bias"); - weight_nodes_.subsampling_pointwise1_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_weight.weights"); - weight_nodes_.subsampling_pointwise1_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise1_bias.bias"); - weight_nodes_.subsampling_depthwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_weight.weights"); - weight_nodes_.subsampling_depthwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_depthwise2_bias.bias"); - weight_nodes_.subsampling_pointwise2_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_weight.weights"); - weight_nodes_.subsampling_pointwise2_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_pointwise2_bias.bias"); - weight_nodes_.subsampling_linear_weight = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_weight.weights"); - weight_nodes_.subsampling_linear_bias = gb->mmap_weights(model_folder_path_ + "/subsampling_linear_bias.bias"); - } - - for (uint32_t i = 0; i < config_.num_layers; ++i) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - - layer.ff1_linear1_weight = gb->mmap_weights(layer_prefix + "ff1_linear1.weights"); - layer.ff1_linear1_bias = gb->mmap_weights(layer_prefix + "ff1_linear1.bias"); - layer.ff1_linear2_weight = gb->mmap_weights(layer_prefix + "ff1_linear2.weights"); - layer.ff1_linear2_bias = gb->mmap_weights(layer_prefix + "ff1_linear2.bias"); - - layer.ff2_linear1_weight = gb->mmap_weights(layer_prefix + "ff2_linear1.weights"); - layer.ff2_linear1_bias = gb->mmap_weights(layer_prefix + "ff2_linear1.bias"); - layer.ff2_linear2_weight = gb->mmap_weights(layer_prefix + "ff2_linear2.weights"); - layer.ff2_linear2_bias = gb->mmap_weights(layer_prefix + "ff2_linear2.bias"); - - layer.self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.self_attn_k_bias = gb->mmap_weights(layer_prefix + "self_attn_k.bias"); - layer.self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - layer.self_attn_relative_k_weight = gb->mmap_weights(layer_prefix + "self_attn_relative_k.weights"); - layer.self_attn_bias_u = gb->mmap_weights(layer_prefix + "self_attn_bias_u.weights"); - layer.self_attn_bias_v = gb->mmap_weights(layer_prefix + "self_attn_bias_v.weights"); - - layer.norm_ff1_weight = gb->mmap_weights(layer_prefix + "norm_ff1.weights"); - layer.norm_ff1_bias = gb->mmap_weights(layer_prefix + "norm_ff1.bias"); - layer.norm_self_attn_weight = gb->mmap_weights(layer_prefix + "norm_self_attn.weights"); - layer.norm_self_attn_bias = gb->mmap_weights(layer_prefix + "norm_self_attn.bias"); - layer.norm_conv_weight = gb->mmap_weights(layer_prefix + "norm_conv.weights"); - layer.norm_conv_bias = gb->mmap_weights(layer_prefix + "norm_conv.bias"); - layer.norm_ff2_weight = gb->mmap_weights(layer_prefix + "norm_ff2.weights"); - layer.norm_ff2_bias = gb->mmap_weights(layer_prefix + "norm_ff2.bias"); - layer.norm_out_weight = gb->mmap_weights(layer_prefix + "norm_out.weights"); - layer.norm_out_bias = gb->mmap_weights(layer_prefix + "norm_out.bias"); - - layer.conv_pointwise1_weight = gb->mmap_weights(layer_prefix + "conv_pointwise1.weights"); - layer.conv_pointwise1_bias = gb->mmap_weights(layer_prefix + "conv_pointwise1.bias"); - layer.conv_depthwise_weight = gb->mmap_weights(layer_prefix + "conv_depthwise.weights"); - layer.conv_depthwise_bias = gb->mmap_weights(layer_prefix + "conv_depthwise.bias"); - layer.conv_pointwise2_weight = gb->mmap_weights(layer_prefix + "conv_pointwise2.weights"); - layer.conv_pointwise2_bias = gb->mmap_weights(layer_prefix + "conv_pointwise2.bias"); - layer.conv_batchnorm_weight = gb->mmap_weights(layer_prefix + "conv_batchnorm_weight.weights"); - layer.conv_batchnorm_bias = gb->mmap_weights(layer_prefix + "conv_batchnorm_bias.bias"); - layer.conv_batchnorm_running_mean = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_mean.weights"); - layer.conv_batchnorm_running_var = gb->mmap_weights(layer_prefix + "conv_batchnorm_running_var.weights"); - } -} - -size_t ParakeetTDTModel::build_subsampling(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> features_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), features_f16.data(), time_major.size()); - - size_t x = gb->input({1, 1, frames, num_mels}, Precision::FP16); - gb->set_input(x, features_f16.data(), Precision::FP16); - - x = gb->conv2d_k3s2p1(x, weight_nodes_.subsampling_conv0_weight, weight_nodes_.subsampling_conv0_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise1_weight, weight_nodes_.subsampling_depthwise1_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise1_weight, weight_nodes_.subsampling_pointwise1_bias); - x = gb->relu(x); - - x = gb->conv2d_depthwise_k3s2p1(x, weight_nodes_.subsampling_depthwise2_weight, weight_nodes_.subsampling_depthwise2_bias); - x = gb->conv2d_pointwise_1x1(x, weight_nodes_.subsampling_pointwise2_weight, weight_nodes_.subsampling_pointwise2_bias); - x = gb->relu(x); - - const auto& conv_shape = gb->get_output_buffer(x).shape; - if (conv_shape.size() != 4 || conv_shape[0] != 1) { - throw std::runtime_error("ParakeetTDT subsampling produced invalid shape"); - } - - const size_t C = conv_shape[1]; - const size_t T = conv_shape[2]; - const size_t W = conv_shape[3]; - - size_t t_major = gb->transposeN(x, {0, 2, 1, 3}, ComputeBackend::CPU); - size_t flattened = gb->reshape(t_major, {T, C * W}); - size_t projected = gb->matmul(flattened, weight_nodes_.subsampling_linear_weight, true, ComputeBackend::CPU); - projected = gb->add(projected, weight_nodes_.subsampling_linear_bias); - return projected; -} - -size_t ParakeetTDTModel::build_relative_position_embeddings(CactusGraph* gb, size_t seq_len) { - const size_t hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - const size_t half_dim = hidden_dim / 2; - const size_t rel_len = 2 * seq_len - 1; - - std::vector pos_embed(rel_len * hidden_dim, 0.0f); - for (size_t p = 0; p < rel_len; ++p) { - const int rel_pos = static_cast(seq_len - 1) - static_cast(p); - for (size_t i = 0; i < half_dim; ++i) { - const float exponent = static_cast(2 * i) / static_cast(hidden_dim); - const float inv_freq = 1.0f / std::pow(10000.0f, exponent); - const float angle = static_cast(rel_pos) * inv_freq; - pos_embed[p * hidden_dim + 2 * i] = std::sin(angle); - if (2 * i + 1 < hidden_dim) { - pos_embed[p * hidden_dim + 2 * i + 1] = std::cos(angle); - } - } - } - - std::vector<__fp16> pos_embed_f16(pos_embed.size()); - cactus_fp32_to_fp16(pos_embed.data(), pos_embed_f16.data(), pos_embed.size()); - - size_t pos_node = gb->input({rel_len, hidden_dim}, Precision::FP16); - gb->set_input(pos_node, pos_embed_f16.data(), Precision::FP16); - return pos_node; -} - -size_t ParakeetTDTModel::build_self_attention(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(hidden, layer.self_attn_q_weight, true, backend); - q = gb->add(q, layer.self_attn_q_bias); - size_t k = gb->matmul(hidden, layer.self_attn_k_weight, true, backend); - k = gb->add(k, layer.self_attn_k_bias); - size_t v = gb->matmul(hidden, layer.self_attn_v_weight, true, backend); - v = gb->add(v, layer.self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT self-attention expects [T, D]"); - } - - const size_t T = q_shape[0]; - const size_t q_heads = std::max(1, static_cast(config_.attention_heads)); - const size_t kv_heads = std::max(1, static_cast(config_.attention_kv_heads)); - const size_t head_dim = std::max(1, static_cast(config_.attention_head_dim)); - - size_t q4 = gb->reshape(q, {1, T, q_heads, head_dim}); - size_t k4 = gb->reshape(k, {1, T, kv_heads, head_dim}); - size_t v4 = gb->reshape(v, {1, T, kv_heads, head_dim}); - - size_t bias_u = gb->reshape(layer.self_attn_bias_u, {1, static_cast(1), q_heads, head_dim}); - size_t bias_v = gb->reshape(layer.self_attn_bias_v, {1, static_cast(1), q_heads, head_dim}); - size_t q_u4 = gb->add(q4, bias_u); - size_t q_v4 = gb->add(q4, bias_v); - - size_t rel_k_flat = gb->matmul(position_embeddings, layer.self_attn_relative_k_weight, true, backend); - size_t rel_k4 = gb->reshape(rel_k_flat, {1, 2 * T - 1, q_heads, head_dim}); - size_t rel_bias = gb->rel_pos_bias(q_v4, rel_k4, attention_scale_); - - size_t attn = gb->attention_masked(q_u4, k4, v4, rel_bias, attention_scale_, false, backend, true); - attn = gb->reshape(attn, {T, q_heads * head_dim}); - - size_t out = gb->matmul(attn, layer.self_attn_output_weight, true, backend); - out = gb->add(out, layer.self_attn_output_bias); - return out; -} - -size_t ParakeetTDTModel::build_feed_forward(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - bool second_ff, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t w1 = second_ff ? layer.ff2_linear1_weight : layer.ff1_linear1_weight; - size_t b1 = second_ff ? layer.ff2_linear1_bias : layer.ff1_linear1_bias; - size_t w2 = second_ff ? layer.ff2_linear2_weight : layer.ff1_linear2_weight; - size_t b2 = second_ff ? layer.ff2_linear2_bias : layer.ff1_linear2_bias; - - size_t x = gb->matmul(hidden, w1, true, backend); - x = gb->add(x, b1); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->matmul(x, w2, true, backend); - x = gb->add(x, b2); - return x; -} - -size_t ParakeetTDTModel::build_convolution_module(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend) { - (void)backend; - const auto& layer = weight_nodes_.layers[layer_idx]; - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT convolution module expects [T, D]"); - } - - const size_t T = hidden_shape[0]; - const size_t D = hidden_shape[1]; - - size_t x = gb->reshape(hidden, {1, T, D}); - x = gb->conv1d_pointwise(x, layer.conv_pointwise1_weight, layer.conv_pointwise1_bias); - x = gb->glu(x, -1); - - x = gb->conv1d_same_depthwise_k9(x, layer.conv_depthwise_weight, layer.conv_depthwise_bias); - - x = gb->batchnorm( - x, - layer.conv_batchnorm_weight, - layer.conv_batchnorm_bias, - layer.conv_batchnorm_running_mean, - layer.conv_batchnorm_running_var, - 2, - 1e-5f - ); - - std::string act = config_.encoder_hidden_act; - std::transform(act.begin(), act.end(), act.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (act.find("gelu") != std::string::npos) { - x = gb->gelu(x); - } else if (act == "relu") { - x = gb->relu(x); - } else { - x = gb->silu(x); - } - - x = gb->conv1d_pointwise(x, layer.conv_pointwise2_weight, layer.conv_pointwise2_bias); - - x = gb->reshape(x, {T, D}); - return x; -} - -size_t ParakeetTDTModel::build_encoder_block(CactusGraph* gb, size_t hidden, size_t position_embeddings, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ff1_in = gb->layernorm(hidden, layer.norm_ff1_weight, layer.norm_ff1_bias); - size_t ff1 = build_feed_forward(gb, ff1_in, layer_idx, false, backend); - ff1 = gb->scalar_multiply(ff1, 0.5f); - size_t x = gb->add(hidden, ff1); - - size_t attn_in = gb->layernorm(x, layer.norm_self_attn_weight, layer.norm_self_attn_bias); - size_t attn = build_self_attention(gb, attn_in, position_embeddings, layer_idx, backend); - x = gb->add(x, attn); - - size_t conv_in = gb->layernorm(x, layer.norm_conv_weight, layer.norm_conv_bias); - size_t conv = build_convolution_module(gb, conv_in, layer_idx, backend); - x = gb->add(x, conv); - - size_t ff2_in = gb->layernorm(x, layer.norm_ff2_weight, layer.norm_ff2_bias); - size_t ff2 = build_feed_forward(gb, ff2_in, layer_idx, true, backend); - ff2 = gb->scalar_multiply(ff2, 0.5f); - x = gb->add(x, ff2); - - x = gb->layernorm(x, layer.norm_out_weight, layer.norm_out_bias); - return x; -} - -size_t ParakeetTDTModel::build_encoder(CactusGraph* gb, const std::vector& audio_features) { - const size_t num_mels = std::max(1, static_cast(config_.num_mel_bins)); - if (audio_features.empty() || (audio_features.size() % num_mels) != 0) { - throw std::runtime_error("ParakeetTDT expects audio_features with shape [num_mels, num_frames]"); - } - - const size_t frames = audio_features.size() / num_mels; - - size_t expected_hidden_dim = std::max(1, static_cast(config_.hidden_dim)); - { - const auto& joint_enc_shape = gb->get_output_buffer(weight_nodes_.joint_enc_weight).shape; - if (joint_enc_shape.size() == 2) { - expected_hidden_dim = joint_enc_shape[1]; - } - } - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector time_major(frames * num_mels); - for (size_t m = 0; m < num_mels; ++m) { - const float* src = &audio_features[m * frames]; - for (size_t t = 0; t < frames; ++t) { - time_major[t * num_mels + m] = src[t]; - } - } - - std::vector<__fp16> time_major_f16(time_major.size()); - cactus_fp32_to_fp16(time_major.data(), time_major_f16.data(), time_major.size()); - - std::vector input_shape = npu_encoder_->get_input_shape(); - std::vector<__fp16> npu_input; - if (pack_parakeet_features_for_npu(time_major_f16, frames, num_mels, input_shape, npu_input)) { - npu_encoder_->preallocate(input_shape, "x", ""); - - std::vector output_shape = npu_encoder_->get_output_shape(); - size_t output_capacity = npu_encoder_->get_output_buffer_size(); - if (output_capacity == 0) { - output_capacity = shape_elements(output_shape); - } - if (output_capacity == 0) { - output_capacity = std::max(1, frames * expected_hidden_dim); - } - - std::vector<__fp16> npu_output(output_capacity); - const size_t elements_written = npu_encoder_->encode( - npu_input.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - size_t T_enc = 0; - size_t D_enc = 0; - if (elements_written > 0 && - infer_npu_encoder_output_shape(output_shape, elements_written, expected_hidden_dim, T_enc, D_enc)) { - const __fp16* src = npu_output.data(); - __fp16* cached_output = npu_encoder_->get_output_buffer(); - const size_t cached_count = npu_encoder_->get_output_buffer_size(); - const size_t required = T_enc * D_enc; - if (cached_output != nullptr && cached_count >= required) { - src = cached_output; - } - - if (D_enc == expected_hidden_dim) { - size_t enc_output = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output, src, Precision::FP16); - return enc_output; - } - } - } - } - - if (!has_cpu_encoder_weights_) { - throw std::runtime_error( - "Parakeet-TDT requires either CPU encoder weights or model.mlpackage encoder output."); - } - - ComputeBackend backend = ComputeBackend::CPU; - size_t hidden = build_subsampling(gb, audio_features); - const auto& hidden_shape = gb->get_output_buffer(hidden).shape; - if (hidden_shape.size() != 2) { - throw std::runtime_error("ParakeetTDT encoder expects subsampling output [T, D]"); - } - - size_t position_embeddings = build_relative_position_embeddings(gb, hidden_shape[0]); - for (uint32_t i = 0; i < config_.num_layers; ++i) { - hidden = build_encoder_block(gb, hidden, position_embeddings, i, backend); - } - - return hidden; -} - -size_t ParakeetTDTModel::forward(const std::vector& audio_features, - const std::vector& tokens, - bool use_cache) { - (void)tokens; - (void)use_cache; - - auto* gb = static_cast(graph_handle_); - gb->clear_debug_nodes(); - return build_encoder(gb, audio_features); -} - -std::vector ParakeetTDTModel::decode_tdt_tokens_with_state( - CactusGraph* gb, - size_t encoder_hidden_node, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState* stream_state, - size_t* out_confirmed_count, - double* out_raw_decoder_time_ms) const { - const auto& enc_buf = gb->get_output_buffer(encoder_hidden_node); - if (enc_buf.shape.size() != 2) { - throw std::runtime_error("ParakeetTDT encoder output must be rank-2 [T, D]"); - } - - const size_t T = enc_buf.shape[0]; - const size_t D = enc_buf.shape[1]; - if (T == 0 || D == 0) { - return {}; - } - - std::vector<__fp16> encoder_fp16 = copy_buffer_to_fp16(enc_buf); - - gb->soft_reset(); - - size_t frame_in = gb->input({1, D}, Precision::FP16); - size_t token_idx = gb->input({1}, Precision::FP32); - - size_t pred = gb->embedding(weight_nodes_.predictor_embed, token_idx); - - const size_t predictor_layers = weight_nodes_.predictor_layers.size(); - std::vector h_prev_nodes; - std::vector c_prev_nodes; - std::vector h_new_nodes; - std::vector c_new_nodes; - std::vector bias_hh_zero_nodes; - std::vector hidden_sizes; - - h_prev_nodes.reserve(predictor_layers); - c_prev_nodes.reserve(predictor_layers); - h_new_nodes.reserve(predictor_layers); - c_new_nodes.reserve(predictor_layers); - bias_hh_zero_nodes.reserve(predictor_layers); - hidden_sizes.reserve(predictor_layers); - - for (size_t i = 0; i < predictor_layers; ++i) { - const auto w_ih = weight_nodes_.predictor_layers[i].weight_ih; - const auto w_hh = weight_nodes_.predictor_layers[i].weight_hh; - const auto b_ih = weight_nodes_.predictor_layers[i].bias; - - if (gb->get_output_buffer(w_ih).precision != Precision::FP16 || - gb->get_output_buffer(w_hh).precision != Precision::FP16 || - gb->get_output_buffer(b_ih).precision != Precision::FP16) { - throw std::runtime_error( - "ParakeetTDT predictor LSTM weights must be FP16. " - "Re-convert the model with updated converter."); - } - - const auto& w_ih_shape = gb->get_output_buffer(w_ih).shape; - if (w_ih_shape.size() != 2 || (w_ih_shape[0] % 4) != 0) { - throw std::runtime_error("ParakeetTDT predictor LSTM weight_ih must be [4*hidden, input]"); - } - const size_t hidden_size = w_ih_shape[0] / 4; - hidden_sizes.push_back(hidden_size); - - size_t h_prev = gb->input({1, hidden_size}, Precision::FP16); - size_t c_prev = gb->input({1, hidden_size}, Precision::FP16); - size_t b_hh_zero = gb->input({4 * hidden_size}, Precision::FP16); - - size_t lstm_out = gb->lstm_cell(pred, h_prev, c_prev, w_ih, w_hh, b_ih, b_hh_zero); - size_t h_new = gb->slice(lstm_out, 2, 0, 1); - size_t c_new = gb->slice(lstm_out, 2, 1, 1); - - h_new = gb->reshape(h_new, {1, hidden_size}); - c_new = gb->reshape(c_new, {1, hidden_size}); - - pred = h_new; - - h_prev_nodes.push_back(h_prev); - c_prev_nodes.push_back(c_prev); - h_new_nodes.push_back(h_new); - c_new_nodes.push_back(c_new); - bias_hh_zero_nodes.push_back(b_hh_zero); - } - - size_t enc_proj = gb->matmul(frame_in, weight_nodes_.joint_enc_weight, true, ComputeBackend::CPU); - enc_proj = gb->add(enc_proj, weight_nodes_.joint_enc_bias); - - size_t pred_proj = gb->matmul(pred, weight_nodes_.joint_pred_weight, true, ComputeBackend::CPU); - pred_proj = gb->add(pred_proj, weight_nodes_.joint_pred_bias); - - size_t joint = gb->add(enc_proj, pred_proj); - joint = gb->relu(joint); - - size_t logits = gb->matmul(joint, weight_nodes_.joint_out_weight, true, ComputeBackend::CPU); - logits = gb->add(logits, weight_nodes_.joint_out_bias); - - const auto& logits_shape = gb->get_output_buffer(logits).shape; - if (logits_shape.size() != 2 || logits_shape[0] != 1) { - throw std::runtime_error("ParakeetTDT joint output must be [1, classes]"); - } - - const size_t total_classes = logits_shape[1]; - std::vector durations = config_.tdt_durations; - size_t duration_classes = durations.size(); - if (duration_classes == 0) { - const size_t fallback = static_cast(config_.tdt_num_durations); - if (fallback > 0) { - durations.resize(fallback); - for (size_t i = 0; i < fallback; ++i) { - durations[i] = static_cast(i); - } - duration_classes = fallback; - } - } - - if (duration_classes == 0 || duration_classes >= total_classes) { - throw std::runtime_error("ParakeetTDT duration classes are invalid"); - } - - const size_t token_classes = total_classes - duration_classes; - - uint32_t blank_id = config_.tdt_blank_id; - if (blank_id >= token_classes) { - const auto& emb_shape = gb->get_output_buffer(weight_nodes_.predictor_embed).shape; - if (emb_shape.size() == 2 && emb_shape[0] > 0) { - const size_t inferred_blank = emb_shape[0] - 1; - blank_id = static_cast(std::min(inferred_blank, token_classes - 1)); - } else { - blank_id = static_cast(token_classes - 1); - } - } - - const size_t time_limit = std::min(end_frame, T); - const size_t emit_begin = std::min(start_frame, time_limit); - const size_t time_begin = std::min(replay_start_frame, emit_begin); - - std::vector> h_state(predictor_layers); - std::vector> c_state(predictor_layers); - std::vector> bias_hh_zero_state(predictor_layers); - const bool can_resume_stream_state = - stream_state && - stream_state->initialized && - stream_state->h.size() == predictor_layers && - stream_state->c.size() == predictor_layers; - - for (size_t i = 0; i < predictor_layers; ++i) { - const bool can_resume_layer = - can_resume_stream_state && - stream_state->h[i].size() == hidden_sizes[i] && - stream_state->c[i].size() == hidden_sizes[i]; - if (can_resume_layer) { - h_state[i] = stream_state->h[i]; - c_state[i] = stream_state->c[i]; - } else { - h_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); - c_state[i].assign(hidden_sizes[i], static_cast<__fp16>(0.0f)); - } - bias_hh_zero_state[i].assign(4 * hidden_sizes[i], static_cast<__fp16>(0.0f)); - gb->set_input(bias_hh_zero_nodes[i], bias_hh_zero_state[i].data(), Precision::FP16); - } - - constexpr float kHopSec = 160.0f / 16000.0f; - const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); - auto* tokenizer = get_tokenizer(); - - std::vector output_tokens; - output_tokens.reserve(T * 2); - - uint32_t last_token = blank_id; - if (can_resume_stream_state && - stream_state->last_token < token_classes) { - last_token = stream_state->last_token; - } - constexpr size_t kMaxSymbolsPerStep = 10; - size_t time_idx = time_begin; - const bool is_stream_mode = cactus::telemetry::isStreamMode(); - std::vector> snap_h = h_state; - std::vector> snap_c = c_state; - uint32_t snap_last_token = last_token; - size_t confirmed_count = 0; - double raw_decoder_time_ms = 0.0; - - while (time_idx < time_limit) { - bool advanced = false; - size_t symbols_added = 0; - - while (symbols_added < kMaxSymbolsPerStep) { - const __fp16* frame_ptr = encoder_fp16.data() + time_idx * D; - gb->set_input(frame_in, frame_ptr, Precision::FP16); - - float token_value = static_cast(last_token); - gb->set_input(token_idx, &token_value, Precision::FP32); - - for (size_t i = 0; i < predictor_layers; ++i) { - gb->set_input(h_prev_nodes[i], h_state[i].data(), Precision::FP16); - gb->set_input(c_prev_nodes[i], c_state[i].data(), Precision::FP16); - } - - const auto decoder_step_start = std::chrono::steady_clock::now(); - gb->execute(); - const auto decoder_step_end = std::chrono::steady_clock::now(); - raw_decoder_time_ms += - std::chrono::duration_cast( - decoder_step_end - decoder_step_start).count() / 1000.0; - - const auto& logits_buf = gb->get_output_buffer(logits); - const auto& bias = get_vocab_bias(); - const size_t best_token = argmax_range_with_bias(logits_buf, 0, token_classes, bias); - const size_t best_duration_idx = argmax_range(logits_buf, token_classes, duration_classes); - const uint32_t predicted_skip = durations[best_duration_idx]; - uint32_t skip = predicted_skip; - if (is_stream_mode && skip > kMaxStreamDurationSkipFrames) { - skip = kMaxStreamDurationSkipFrames; - } - - if (best_token != blank_id) { - if (stream_state && time_idx >= emit_begin && tokenizer) { - std::string piece = tokenizer->decode({static_cast(best_token)}); - if (!piece.empty() && piece[0] == ' ' && !output_tokens.empty()) { - snap_h = h_state; - snap_c = c_state; - snap_last_token = last_token; - confirmed_count = output_tokens.size(); - } - } - if (time_idx >= emit_begin) { - output_tokens.push_back( - {static_cast(best_token), time_idx * frame_sec, (time_idx + skip) * frame_sec}); - } - last_token = static_cast(best_token); - - for (size_t i = 0; i < predictor_layers; ++i) { - const auto& h_buf = gb->get_output_buffer(h_new_nodes[i]); - const auto& c_buf = gb->get_output_buffer(c_new_nodes[i]); - const __fp16* h_ptr = h_buf.data_as<__fp16>(); - const __fp16* c_ptr = c_buf.data_as<__fp16>(); - std::copy(h_ptr, h_ptr + hidden_sizes[i], h_state[i].begin()); - std::copy(c_ptr, c_ptr + hidden_sizes[i], c_state[i].begin()); - } - } - - ++symbols_added; - - if (skip > 0) { - time_idx += skip; - advanced = true; - break; - } - - if (best_token == blank_id) { - ++time_idx; - advanced = true; - break; - } - } - - if (!advanced) { - ++time_idx; - } - } - - if (out_confirmed_count) { - *out_confirmed_count = confirmed_count; - } - if (out_raw_decoder_time_ms) { - *out_raw_decoder_time_ms = raw_decoder_time_ms; - } - - if (stream_state) { - stream_state->initialized = true; - stream_state->last_token = snap_last_token; - stream_state->h = std::move(snap_h); - stream_state->c = std::move(snap_c); - } - - return output_tokens; -} - -std::vector ParakeetTDTModel::greedy_decode_tdt_tokens( - CactusGraph* gb, - size_t encoder_hidden_node) const { - return decode_tdt_tokens_with_state( - gb, - encoder_hidden_node, - 0, - 0, - std::numeric_limits::max(), - nullptr); -} - -uint32_t ParakeetTDTModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* out_token_time_start, - float* out_token_time_end) -{ - (void)temperature; - (void)top_p; - (void)top_k; - (void)min_p; - (void)repetition_penalty; - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_with_audio"); - } - - auto* gb = static_cast(graph_handle_); - - const bool new_request = !tdt_tokens_ready_ || tokens.empty() || tokens.size() < last_input_token_count_; - if (new_request) { - gb->soft_reset(); - size_t encoder_out = forward(audio_features, tokens, false); - - if (!profile_file.empty()) { - gb->execute(profile_file); - } else { - gb->execute(); - } - - tdt_tokens_ = greedy_decode_tdt_tokens(gb, encoder_out); - tdt_emit_index_ = 0; - tdt_tokens_ready_ = true; - } - - last_input_token_count_ = tokens.size(); - if (out_entropy) { - *out_entropy = 0.0f; - } - - if (tdt_emit_index_ < tdt_tokens_.size()) { - const auto& tok = tdt_tokens_[tdt_emit_index_++]; - if (out_token_time_start) *out_token_time_start = tok.time_start; - if (out_token_time_end) *out_token_time_end = tok.time_end; - return tok.id; - } - - return get_tokenizer()->get_eos_token(); -} - -ParakeetTDTModel::ChunkStreamResult ParakeetTDTModel::decode_chunk_stream( - const std::vector& audio_features, - size_t replay_start_frame, - size_t start_frame, - size_t end_frame, - ChunkStreamState& state) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (audio_features.empty()) { - throw std::runtime_error("Audio features cannot be empty in ParakeetTDT decode_chunk_stream"); - } - - auto* gb = static_cast(graph_handle_); - - gb->soft_reset(); - size_t encoder_out = forward(audio_features, {}, false); - gb->execute(); - - ChunkStreamResult result; - double raw_decoder_time_ms = 0.0; - std::vector tokens = decode_tdt_tokens_with_state( - gb, encoder_out, replay_start_frame, start_frame, end_frame, &state, - &result.confirmed_token_count, &raw_decoder_time_ms); - - result.token_count = tokens.size(); - result.raw_decoder_time_ms = raw_decoder_time_ms; - result.raw_decoder_tps = - (result.token_count > 0 && raw_decoder_time_ms > 0.0) - ? (static_cast(result.token_count) * 1000.0) / raw_decoder_time_ms - : 0.0; - constexpr float kHopSec = 160.0f / 16000.0f; - const float frame_sec = kHopSec * static_cast(config_.subsampling_factor); - result.start_sec = start_frame * frame_sec; - result.confirmed_end_sec = start_frame * frame_sec; - result.resume_end_sec = replay_start_frame * frame_sec; - result.end_sec = start_frame * frame_sec; - if (!tokens.empty()) { - result.start_sec = tokens.front().time_start; - result.end_sec = tokens.back().time_end; - if (result.confirmed_token_count > 0 && - result.confirmed_token_count <= tokens.size()) { - result.confirmed_end_sec = tokens[result.confirmed_token_count - 1].time_end; - result.resume_end_sec = result.confirmed_end_sec; - } - } - - auto* tokenizer = get_tokenizer(); - if (!tokenizer) { - throw std::runtime_error("Tokenizer unavailable in ParakeetTDT decode_chunk_stream"); - } - for (size_t i = 0; i < tokens.size(); ++i) { - const std::string piece = tokenizer->decode({tokens[i].id}); - result.text += piece; - if (i < result.confirmed_token_count) { - result.confirmed_text += piece; - } else { - result.pending_text += piece; - } - } - return result; -} - -std::vector ParakeetTDTModel::get_audio_embeddings(const std::vector& audio_features) { - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - size_t hidden = build_encoder(gb, audio_features); - size_t pooled = gb->mean(hidden, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - const size_t hidden_dim = output_buf.total_size; - std::vector embedding(hidden_dim); - - if (output_buf.precision == Precision::FP32) { - const float* src = output_buf.data_as(); - std::copy(src, src + hidden_dim, embedding.begin()); - } else if (output_buf.precision == Precision::FP16) { - const __fp16* src = output_buf.data_as<__fp16>(); - Quantization::fp16_to_fp32(src, embedding.data(), hidden_dim); - } else { - const int8_t* src = output_buf.data_as(); - Quantization::int8_to_fp32(src, embedding.data(), hidden_dim, 1.0f); - } - - reset_cache(); - return embedding; -} - -void ParakeetTDTModel::reset_cache() { - Model::reset_cache(); - tdt_tokens_ready_ = false; - tdt_emit_index_ = 0; - tdt_tokens_.clear(); - last_input_token_count_ = 0; -} - -} // namespace engine -} // namespace cactus diff --git a/cactus/models/model_pyannote.cpp b/cactus/models/model_pyannote.cpp deleted file mode 100644 index 44ac4c8c3..000000000 --- a/cactus/models/model_pyannote.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include - -namespace cactus { -namespace engine { - -static constexpr size_t WINDOW_SAMPLES = 160000; -static constexpr size_t FRAMES_PER_WINDOW = 589; -static constexpr int NUM_CLASSES = 7; -static constexpr int NUM_SPEAKERS = 3; - -PyAnnoteModel::PyAnnoteModel() : Model() {} -PyAnnoteModel::PyAnnoteModel(const Config& config) : Model(config) {} - -void PyAnnoteModel::load_weights_to_graph(CactusGraph* gb) { - const std::string& p = model_folder_path_; - - weight_nodes_.sinc_filters = gb->mmap_weights(p + "/sincnet_sinc_filters.weights"); - weight_nodes_.wav_norm_weight = gb->mmap_weights(p + "/sincnet_wav_norm_weight.weights"); - weight_nodes_.wav_norm_bias = gb->mmap_weights(p + "/sincnet_wav_norm_bias.weights"); - weight_nodes_.norm0_weight = gb->mmap_weights(p + "/sincnet_norm0_weight.weights"); - weight_nodes_.norm0_bias = gb->mmap_weights(p + "/sincnet_norm0_bias.weights"); - weight_nodes_.conv1_weight = gb->mmap_weights(p + "/sincnet_conv1_weight.weights"); - weight_nodes_.conv1_bias = gb->mmap_weights(p + "/sincnet_conv1_bias.weights"); - weight_nodes_.norm1_weight = gb->mmap_weights(p + "/sincnet_norm1_weight.weights"); - weight_nodes_.norm1_bias = gb->mmap_weights(p + "/sincnet_norm1_bias.weights"); - weight_nodes_.conv2_weight = gb->mmap_weights(p + "/sincnet_conv2_weight.weights"); - weight_nodes_.conv2_bias = gb->mmap_weights(p + "/sincnet_conv2_bias.weights"); - weight_nodes_.norm2_weight = gb->mmap_weights(p + "/sincnet_norm2_weight.weights"); - weight_nodes_.norm2_bias = gb->mmap_weights(p + "/sincnet_norm2_bias.weights"); - - for (int layer = 0; layer < 4; ++layer) { - auto& lw = weight_nodes_.lstm_layers[layer]; - std::string fwd = p + "/lstm_fwd_" + std::to_string(layer); - std::string bwd = p + "/lstm_bwd_" + std::to_string(layer); - lw.w_ih_fwd = gb->mmap_weights(fwd + "_weight_ih.weights"); - lw.w_hh_fwd = gb->mmap_weights(fwd + "_weight_hh.weights"); - lw.b_ih_fwd = gb->mmap_weights(fwd + "_bias_ih.weights"); - lw.b_hh_fwd = gb->mmap_weights(fwd + "_bias_hh.weights"); - lw.w_ih_bwd = gb->mmap_weights(bwd + "_weight_ih.weights"); - lw.w_hh_bwd = gb->mmap_weights(bwd + "_weight_hh.weights"); - lw.b_ih_bwd = gb->mmap_weights(bwd + "_bias_ih.weights"); - lw.b_hh_bwd = gb->mmap_weights(bwd + "_bias_hh.weights"); - } - - weight_nodes_.linear0_weight = gb->mmap_weights(p + "/linear_0_weight.weights"); - weight_nodes_.linear0_bias = gb->mmap_weights(p + "/linear_0_bias.weights"); - weight_nodes_.linear1_weight = gb->mmap_weights(p + "/linear_1_weight.weights"); - weight_nodes_.linear1_bias = gb->mmap_weights(p + "/linear_1_bias.weights"); - weight_nodes_.classifier_weight = gb->mmap_weights(p + "/classifier_weight.weights"); - weight_nodes_.classifier_bias = gb->mmap_weights(p + "/classifier_bias.weights"); -} - -void PyAnnoteModel::build_graph() { - audio_input_ = graph_.input({1, 1, 160000}, Precision::FP16); - - size_t x = graph_.groupnorm(audio_input_, weight_nodes_.wav_norm_weight, weight_nodes_.wav_norm_bias, 1, 1e-5f); - x = graph_.conv1d(x, weight_nodes_.sinc_filters, 10); - x = graph_.abs(x); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm0_weight, weight_nodes_.norm0_bias, 80, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.conv1d(x, weight_nodes_.conv1_weight, weight_nodes_.conv1_bias, 1); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm1_weight, weight_nodes_.norm1_bias, 60, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.conv1d(x, weight_nodes_.conv2_weight, weight_nodes_.conv2_bias, 1); - x = graph_.maxpool1d(x, 3, 3); - x = graph_.groupnorm(x, weight_nodes_.norm2_weight, weight_nodes_.norm2_bias, 60, 1e-5f); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.transposeN(x, {0, 2, 1}); - - for (int layer = 0; layer < 4; ++layer) { - auto& lw = weight_nodes_.lstm_layers[layer]; - x = graph_.bilstm_sequence(x, - lw.w_ih_fwd, lw.w_hh_fwd, lw.b_ih_fwd, lw.b_hh_fwd, - lw.w_ih_bwd, lw.w_hh_bwd, lw.b_ih_bwd, lw.b_hh_bwd); - } - - const auto& bilstm_shape = graph_.get_output_buffer(x).shape; - size_t T = bilstm_shape[1]; - x = graph_.reshape(x, {T, bilstm_shape[2]}); - - x = graph_.add(graph_.matmul(x, weight_nodes_.linear0_weight, true), weight_nodes_.linear0_bias); - x = graph_.leaky_relu(x, 0.01f); - x = graph_.add(graph_.matmul(x, weight_nodes_.linear1_weight, true), weight_nodes_.linear1_bias); - x = graph_.leaky_relu(x, 0.01f); - - x = graph_.add(graph_.matmul(x, weight_nodes_.classifier_weight, true), weight_nodes_.classifier_bias); - x = graph_.softmax(x, -1); - x = graph_.reshape(x, {1, T, 7}); - - output_node_ = x; -} - -bool PyAnnoteModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - build_graph(); - - chunk_buf_.resize(WINDOW_SAMPLES); - - hamming_.resize(FRAMES_PER_WINDOW); - for (size_t i = 0; i < FRAMES_PER_WINDOW; ++i) - hamming_[i] = 0.54f - 0.46f * std::cos(2.0f * M_PI * i / (FRAMES_PER_WINDOW - 1)); - - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -std::vector PyAnnoteModel::diarize(const float* pcm_f32, size_t num_samples, size_t step_samples, bool raw_powerset) { - if (!initialized_) throw std::runtime_error("PyAnnote model not initialized"); - - const size_t total_frames = std::max( - FRAMES_PER_WINDOW, - static_cast(std::round((double)num_samples * FRAMES_PER_WINDOW / WINDOW_SAMPLES)) - ); - - std::vector aggregated(total_frames * NUM_CLASSES, 0.0f); - std::vector weight_sum(total_frames, 0.0f); - - auto process_chunk = [&](size_t chunk_start) { - const size_t copy_len = std::min(WINDOW_SAMPLES, num_samples - chunk_start); - for (size_t i = 0; i < copy_len; ++i) - chunk_buf_[i] = static_cast<__fp16>(pcm_f32[chunk_start + i]); - std::fill(chunk_buf_.begin() + copy_len, chunk_buf_.end(), static_cast<__fp16>(0.0f)); - - graph_.set_input(audio_input_, chunk_buf_.data(), Precision::FP16); - graph_.execute(); - - const __fp16* chunk_scores = graph_.get_output_buffer(output_node_).data_as<__fp16>(); - const size_t frame_offset = static_cast(std::round((double)chunk_start * FRAMES_PER_WINDOW / WINDOW_SAMPLES)); - - for (size_t f = 0; f < FRAMES_PER_WINDOW; ++f) { - const size_t out_f = frame_offset + f; - if (out_f >= total_frames) break; - const float w = hamming_[f]; - weight_sum[out_f] += w; - for (int c = 0; c < NUM_CLASSES; ++c) - aggregated[out_f * NUM_CLASSES + c] += w * static_cast(chunk_scores[f * NUM_CLASSES + c]); - } - }; - - const size_t last_start = num_samples > WINDOW_SAMPLES ? num_samples - WINDOW_SAMPLES : 0; - bool last_processed = false; - for (size_t s = 0; s + WINDOW_SAMPLES <= num_samples; s += step_samples) { - if (s >= last_start) last_processed = true; - process_chunk(s); - } - if (!last_processed) - process_chunk(last_start); - - for (size_t f = 0; f < total_frames; ++f) { - if (weight_sum[f] > 0.0f) { - const float inv_w = 1.0f / weight_sum[f]; - float* agg = aggregated.data() + f * NUM_CLASSES; - for (int c = 0; c < NUM_CLASSES; ++c) - agg[c] *= inv_w; - } - } - - if (raw_powerset) { - return aggregated; - } - - std::vector speaker_probs(total_frames * NUM_SPEAKERS, 0.0f); - for (size_t f = 0; f < total_frames; ++f) { - const float* agg = aggregated.data() + f * NUM_CLASSES; - float* dst = speaker_probs.data() + f * NUM_SPEAKERS; - dst[0] = agg[1] + agg[4] + agg[5]; - dst[1] = agg[2] + agg[4] + agg[6]; - dst[2] = agg[3] + agg[5] + agg[6]; - } - - return speaker_probs; -} - -} -} diff --git a/cactus/models/model_qwen.cpp b/cactus/models/model_qwen.cpp deleted file mode 100644 index 601c05334..000000000 --- a/cactus/models/model_qwen.cpp +++ /dev/null @@ -1,171 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include - - -namespace cactus { -namespace engine { - -QwenModel::QwenModel() : Model() {} - -QwenModel::QwenModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -void QwenModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - layer.attn_k_weight = gb->mmap_weights(layer_prefix + "attn_k.weights"); - layer.attn_v_weight = gb->mmap_weights(layer_prefix + "attn_v.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.attn_q_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_norm.weights"); - layer.attn_k_norm_weight = gb->mmap_weights(layer_prefix + "attn_k_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t QwenModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, layer.attn_q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, layer.attn_k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - q_proj_4d = gb->rope(q_proj_4d, config_.rope_theta, position_offset); - k_proj_4d = gb->rope(k_proj_4d, config_.rope_theta, position_offset); - } - - size_t attn_output_4d; - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - } - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - - -size_t QwenModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - - -size_t QwenModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add(hidden, attn_output); - auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add(after_attention, mlp_output); -} - - -size_t QwenModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - auto seq_len = static_cast(tokens.size()); - - size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - auto final_hidden = gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); - - return final_hidden; -} - -} -} \ No newline at end of file diff --git a/cactus/models/model_qwen3p5.cpp b/cactus/models/model_qwen3p5.cpp deleted file mode 100644 index 0f438e1c0..000000000 --- a/cactus/models/model_qwen3p5.cpp +++ /dev/null @@ -1,771 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include -#include -#include -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -Qwen3p5Model::Qwen3p5Model() : Model() { - weight_nodes_.layers.resize(config_.num_layers); - conv_cache_state_nodes_.assign(config_.num_layers, 0); -} - -Qwen3p5Model::Qwen3p5Model(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); - conv_cache_state_nodes_.assign(config.num_layers, 0); -} - -void Qwen3p5Model::post_init() { - bool has_deltanet_layer = false; - const WeightNodeIDs::LayerWeights* first_linear = nullptr; - for (const auto& layer : weight_nodes_.layers) { - if (layer.type == WeightNodeIDs::LayerType::DELTANET) { - has_deltanet_layer = true; - if (!first_linear) { - first_linear = &layer.weights; - } - break; - } - } - - deltanet_heads_ = static_cast(config_.attention_heads); - deltanet_key_dim_ = static_cast(config_.attention_head_dim); - deltanet_value_dim_ = static_cast(config_.attention_head_dim); - - if (has_deltanet_layer && - config_.linear_num_key_heads > 0 && - config_.linear_key_head_dim > 0) { - const size_t num_k_heads = static_cast(config_.linear_num_key_heads); - const size_t key_head_dim = static_cast(config_.linear_key_head_dim); - size_t num_v_heads = static_cast(config_.linear_num_value_heads > 0 - ? config_.linear_num_value_heads - : config_.linear_num_key_heads); - size_t value_head_dim = static_cast(config_.linear_value_head_dim); - - if (num_v_heads == 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention value head count in config"); - } - if (value_head_dim == 0) { - if (config_.linear_v_proj_dim > 0 && (config_.linear_v_proj_dim % num_v_heads == 0)) { - value_head_dim = static_cast(config_.linear_v_proj_dim / num_v_heads); - } else { - throw std::runtime_error("Qwen3p5Model cannot derive linear-attention value head dim from config"); - } - } - if (config_.linear_q_proj_dim > 0 && - static_cast(config_.linear_q_proj_dim) != num_k_heads * key_head_dim) { - throw std::runtime_error("Qwen3p5Model linear-attention q projection dim mismatch with key-head topology"); - } - if (config_.linear_v_proj_dim > 0 && - static_cast(config_.linear_v_proj_dim) != num_v_heads * value_head_dim) { - throw std::runtime_error("Qwen3p5Model linear-attention v projection dim mismatch with value-head topology"); - } - - deltanet_heads_ = num_v_heads; - deltanet_key_dim_ = key_head_dim; - deltanet_value_dim_ = value_head_dim; - } else if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention dims: graph unavailable"); - } - if (!first_linear) { - throw std::runtime_error("Qwen3p5Model expected deltanet layer weights for dim inference"); - } - - const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); - const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); - const auto& n_buf = gb->get_output_buffer(first_linear->attn_q_norm_weight); - if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2 || n_buf.shape.empty()) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention weight shapes"); - } - - const size_t q_proj_dim = q_buf.shape[0]; - const size_t v_proj_dim = v_buf.shape[0]; - const size_t inferred_key_dim = n_buf.shape[0]; - if (inferred_key_dim == 0 || q_proj_dim == 0 || v_proj_dim == 0) { - throw std::runtime_error("Qwen3p5Model failed to infer non-zero linear-attention dims"); - } - if (q_proj_dim % inferred_key_dim != 0) { - throw std::runtime_error("Qwen3p5Model q projection dim is not divisible by inferred key dim"); - } - - const size_t inferred_k_heads = q_proj_dim / inferred_key_dim; - const size_t inferred_v_heads = v_proj_dim / inferred_key_dim; - if (inferred_k_heads == 0 || inferred_v_heads == 0 || inferred_v_heads % inferred_k_heads != 0) { - throw std::runtime_error("Qwen3p5Model failed to infer compatible linear-attention head topology"); - } - - deltanet_heads_ = inferred_v_heads; - deltanet_key_dim_ = inferred_key_dim; - deltanet_value_dim_ = inferred_key_dim; - } - - if (deltanet_heads_ == 0 || deltanet_key_dim_ == 0 || deltanet_value_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model requires non-zero deltanet heads/key/value dims"); - } - - deltanet_state_flat_dim_ = deltanet_heads_ * deltanet_key_dim_ * deltanet_value_dim_; - - deltanet_mixed_dim_ = 0; - if (config_.linear_q_proj_dim > 0 && config_.linear_v_proj_dim > 0) { - deltanet_mixed_dim_ = static_cast(2 * config_.linear_q_proj_dim + config_.linear_v_proj_dim); - } else if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb || !first_linear) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention mixed projection dim"); - } - const auto& q_buf = gb->get_output_buffer(first_linear->attn_q_weight); - const auto& v_buf = gb->get_output_buffer(first_linear->attn_v_weight); - if (q_buf.shape.size() < 2 || v_buf.shape.size() < 2) { - throw std::runtime_error("Qwen3p5Model invalid q/v projection shapes for mixed dim inference"); - } - deltanet_mixed_dim_ = (2 * q_buf.shape[0]) + v_buf.shape[0]; - } - - deltanet_conv_history_len_ = 0; - if (has_deltanet_layer) { - auto* gb = static_cast(graph_handle_); - if (!gb) { - throw std::runtime_error("Qwen3p5Model cannot infer linear-attention conv history: graph unavailable"); - } - for (const auto& layer : weight_nodes_.layers) { - if (layer.type != WeightNodeIDs::LayerType::DELTANET || layer.weights.deltanet_conv_weight == 0) { - continue; - } - const auto& conv_buf = gb->get_output_buffer(layer.weights.deltanet_conv_weight); - if (conv_buf.shape.empty()) { - continue; - } - const size_t kernel = conv_buf.shape.back(); - if (kernel > 1) { - deltanet_conv_history_len_ = std::max(deltanet_conv_history_len_, kernel - 1); - } - } - } - - if (deltanet_conv_history_len_ > 0 && deltanet_mixed_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model linear-attention conv history requires non-zero mixed dim"); - } - deltanet_conv_flat_dim_ = deltanet_conv_history_len_ * deltanet_mixed_dim_; - deltanet_cache_row_dim_ = deltanet_state_flat_dim_ + deltanet_conv_flat_dim_; - if (deltanet_cache_row_dim_ == 0) { - throw std::runtime_error("Qwen3p5Model computed zero cache row dim"); - } - - conv_cache_.init(config_.num_layers, deltanet_cache_row_dim_, 1, Precision::FP32); - last_forward_used_cache_ = false; - deltanet_total_seq_len_ = 0; -} - -void Qwen3p5Model::reset_cache() { - Model::reset_cache(); - if (conv_cache_.window_size > 0) { - conv_cache_.reset(); - } - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - deltanet_total_seq_len_ = 0; - last_forward_used_cache_ = false; -} - -void Qwen3p5Model::post_execute_updates(CactusGraph* gb, size_t seq_len) { - if (!last_forward_used_cache_) { - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - return; - } - - const size_t layer_count = std::min(conv_cache_state_nodes_.size(), weight_nodes_.layers.size()); - for (size_t layer_idx = 0; layer_idx < layer_count; ++layer_idx) { - if (weight_nodes_.layers[layer_idx].type != WeightNodeIDs::LayerType::DELTANET) { - conv_cache_state_nodes_[layer_idx] = 0; - continue; - } - - const size_t state_node = conv_cache_state_nodes_[layer_idx]; - if (state_node != 0) { - conv_cache_.update(gb, layer_idx, state_node); - } - conv_cache_state_nodes_[layer_idx] = 0; - } - - deltanet_total_seq_len_ += seq_len; - last_forward_used_cache_ = false; -} - -void Qwen3p5Model::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - auto choose_existing_weight = [&](const std::vector& candidates) -> std::string { - for (const auto& p : candidates) { - if (std::filesystem::exists(p)) { - return p; - } - } - return std::string(); - }; - - auto mmap_required = [&](const std::vector& candidates, const std::string& weight_name) -> size_t { - const std::string path = choose_existing_weight(candidates); - if (path.empty()) { - throw std::runtime_error("Qwen3p5Model missing required weight: " + weight_name); - } - return gb->mmap_weights(path); - }; - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer_entry = weight_nodes_.layers[i]; - auto& layer = layer_entry.weights; - - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - bool is_deltanet = false; - if (i < config_.layer_types.size()) { - std::string lt = config_.layer_types[i]; - std::transform(lt.begin(), lt.end(), lt.begin(), ::tolower); - is_deltanet = (lt.find("deltanet") != std::string::npos || - lt.find("linear_attention") != std::string::npos || - lt.find("linear_attn") != std::string::npos); - } - layer_entry.type = is_deltanet ? WeightNodeIDs::LayerType::DELTANET : WeightNodeIDs::LayerType::ATTENTION; - - layer.attn_q_weight = mmap_required( - {layer_prefix + "attn_q.weights", layer_prefix + "linear_attn_q.weights"}, - "layer_" + std::to_string(i) + "_attn_q/linear_attn_q"); - layer.attn_k_weight = mmap_required( - {layer_prefix + "attn_k.weights", layer_prefix + "linear_attn_k.weights"}, - "layer_" + std::to_string(i) + "_attn_k/linear_attn_k"); - layer.attn_v_weight = mmap_required( - {layer_prefix + "attn_v.weights", layer_prefix + "linear_attn_v.weights"}, - "layer_" + std::to_string(i) + "_attn_v/linear_attn_v"); - layer.attn_output_weight = mmap_required( - {layer_prefix + "attn_output.weights", layer_prefix + "linear_attn_output.weights"}, - "layer_" + std::to_string(i) + "_attn_output/linear_attn_output"); - layer.input_layernorm_weight = mmap_required( - {layer_prefix + "input_norm.weights"}, - "layer_" + std::to_string(i) + "_input_norm"); - layer.attn_q_norm_weight = mmap_required( - {layer_prefix + "attn_q_norm.weights", layer_prefix + "linear_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_attn_q_norm/linear_attn_norm"); - layer.attn_k_norm_weight = mmap_required( - {layer_prefix + "attn_k_norm.weights", layer_prefix + "linear_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_attn_k_norm/linear_attn_norm"); - - if (is_deltanet) { - const std::string qkv_path = choose_existing_weight({ - layer_prefix + "linear_attn_qkv.weights" - }); - layer.deltanet_qkv_weight = qkv_path.empty() ? 0 : gb->mmap_weights(qkv_path); - - const std::string gate_path = choose_existing_weight({ - layer_prefix + "deltanet_gate.weights", - layer_prefix + "attn_gate.weights", - layer_prefix + "attn_f_gate.weights", - layer_prefix + "linear_attn_a.weights" - }); - const std::string beta_path = choose_existing_weight({ - layer_prefix + "deltanet_beta.weights", - layer_prefix + "attn_beta.weights", - layer_prefix + "attn_f_beta.weights", - layer_prefix + "linear_attn_b.weights" - }); - if (gate_path.empty() || beta_path.empty()) { - throw std::runtime_error( - "Qwen3p5Model deltanet layer missing gate/beta weights at layer " + std::to_string(i)); - } - layer.deltanet_gate_weight = gb->mmap_weights(gate_path); - layer.deltanet_beta_weight = gb->mmap_weights(beta_path); - - const std::string gate_bias_path = choose_existing_weight({ - layer_prefix + "linear_attn_A_log.weights" - }); - const std::string beta_bias_path = choose_existing_weight({ - layer_prefix + "linear_attn_dt_bias.weights" - }); - layer.deltanet_gate_bias = gate_bias_path.empty() ? 0 : gb->mmap_weights(gate_bias_path); - layer.deltanet_beta_bias = beta_bias_path.empty() ? 0 : gb->mmap_weights(beta_bias_path); - const std::string z_path = choose_existing_weight({ - layer_prefix + "linear_attn_z.weights" - }); - const std::string conv_path = choose_existing_weight({ - layer_prefix + "linear_attn_conv1d.weights" - }); - layer.deltanet_z_weight = z_path.empty() ? 0 : gb->mmap_weights(z_path); - layer.deltanet_conv_weight = conv_path.empty() ? 0 : gb->mmap_weights(conv_path); - } - - layer.ffn_gate_weight = mmap_required( - {layer_prefix + "ffn_gate.weights"}, - "layer_" + std::to_string(i) + "_ffn_gate"); - layer.ffn_up_weight = mmap_required( - {layer_prefix + "ffn_up.weights"}, - "layer_" + std::to_string(i) + "_ffn_up"); - layer.ffn_down_weight = mmap_required( - {layer_prefix + "ffn_down.weights"}, - "layer_" + std::to_string(i) + "_ffn_down"); - layer.post_attention_layernorm_weight = mmap_required( - {layer_prefix + "post_attn_norm.weights"}, - "layer_" + std::to_string(i) + "_post_attn_norm"); - } - - if (npu::is_npu_available()) { - std::string npu_prefill_path = model_folder_path_ + "/model.mlpackage"; - load_npu_prefill(npu_prefill_path); - } -} - -size_t Qwen3p5Model::build_gated_deltanet(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - (void)position_offset; - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - - const auto& in_shape = gb->get_output_buffer(normalized_input).shape; - if (in_shape.size() < 2) { - throw std::runtime_error("Qwen3p5Model linear-attention expects rank-2 normalized input"); - } - const size_t seq_len = in_shape[0]; - - size_t q_proj = 0; - size_t k_proj = 0; - size_t v_proj = 0; - size_t mixed_qkv = 0; - size_t z_proj = layer.deltanet_z_weight ? gb->matmul(normalized_input, layer.deltanet_z_weight, true, backend) : 0; - - const size_t num_heads = deltanet_heads_; - const size_t key_dim = deltanet_key_dim_; - const size_t value_dim = deltanet_value_dim_; - const size_t v_proj_dim = num_heads * value_dim; - size_t qk_proj_dim = 0; - - if (layer.deltanet_qkv_weight != 0) { - mixed_qkv = gb->matmul(normalized_input, layer.deltanet_qkv_weight, true, backend); - const auto& mixed_shape = gb->get_output_buffer(mixed_qkv).shape; - if (mixed_shape.size() < 2 || mixed_shape[0] != seq_len) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection shape"); - } - if (mixed_shape[1] <= v_proj_dim || ((mixed_shape[1] - v_proj_dim) % 2) != 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention joint qkv projection width"); - } - qk_proj_dim = (mixed_shape[1] - v_proj_dim) / 2; - } else { - q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - if (q_shape.size() < 2 || q_shape[0] != seq_len) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention q projection shape"); - } - qk_proj_dim = q_shape[1]; - size_t mixed_qk = gb->concat(q_proj, k_proj, 1); - mixed_qkv = gb->concat(mixed_qk, v_proj, 1); - } - - if (qk_proj_dim == 0 || qk_proj_dim % key_dim != 0 || v_proj_dim == 0) { - throw std::runtime_error("Qwen3p5Model invalid linear-attention projection dimensions"); - } - const size_t num_k_heads = qk_proj_dim / key_dim; - if (num_heads % num_k_heads != 0) { - throw std::runtime_error("Qwen3p5Model requires num_v_heads divisible by num_k_heads"); - } - - const size_t mixed_proj_dim = qk_proj_dim + qk_proj_dim + v_proj_dim; - if (deltanet_mixed_dim_ != 0 && mixed_proj_dim != deltanet_mixed_dim_) { - throw std::runtime_error("Qwen3p5Model linear-attention mixed projection dim mismatch"); - } - const size_t state_flat_dim = deltanet_state_flat_dim_; - size_t prev_state_flat = 0; - size_t prev_conv_flat = 0; - if (use_cache && conv_cache_.window_size > 0) { - const auto view = conv_cache_.get_window(layer_idx); - if (conv_cache_.window_size == 1 && view.total_len == 1) { - const uint8_t* cache_row = nullptr; - if (view.len2 == 1 && view.ptr2 != nullptr) { - cache_row = static_cast(view.ptr2); - } else if (view.len1 == 1 && view.ptr1 != nullptr) { - cache_row = static_cast(view.ptr1); - } - if (cache_row != nullptr) { - prev_state_flat = gb->input({1, state_flat_dim}, conv_cache_.precision); - gb->set_external_input(prev_state_flat, const_cast(static_cast(cache_row)), - conv_cache_.precision); - if (deltanet_conv_flat_dim_ > 0) { - const size_t state_bytes = state_flat_dim * conv_cache_.element_size; - prev_conv_flat = gb->input({1, deltanet_conv_flat_dim_}, conv_cache_.precision); - gb->set_external_input( - prev_conv_flat, - const_cast(static_cast(cache_row + state_bytes)), - conv_cache_.precision); - } - } - } - - if (prev_state_flat == 0) { - std::vector segments; - if (view.len2 > 0) { - size_t left_node = gb->input({view.len2, deltanet_cache_row_dim_}, conv_cache_.precision); - gb->set_external_input(left_node, const_cast(view.ptr2), conv_cache_.precision); - segments.push_back(left_node); - } - if (view.len1 > 0) { - size_t right_node = gb->input({view.len1, deltanet_cache_row_dim_}, conv_cache_.precision); - gb->set_external_input(right_node, const_cast(view.ptr1), conv_cache_.precision); - segments.push_back(right_node); - } - - if (!segments.empty()) { - size_t stacked = segments[0]; - for (size_t i = 1; i < segments.size(); ++i) { - stacked = gb->concat(stacked, segments[i], 0); - } - if (view.total_len > 1) { - stacked = gb->slice(stacked, 0, view.total_len - 1, 1); - } - prev_state_flat = gb->slice(stacked, 1, 0, state_flat_dim); - if (deltanet_conv_flat_dim_ > 0) { - prev_conv_flat = gb->slice(stacked, 1, state_flat_dim, deltanet_conv_flat_dim_); - } - } - } - } - - size_t conv_source_with_history = mixed_qkv; - if (layer.deltanet_conv_weight) { - const auto& conv_wbuf = gb->get_output_buffer(layer.deltanet_conv_weight); - const size_t kernel = conv_wbuf.shape.back(); - if (kernel > 1 && use_cache && deltanet_conv_history_len_ > 0) { - size_t history_2d = 0; - if (prev_conv_flat != 0) { - history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); - history_2d = gb->precision_cast(history_2d, Precision::FP16); - } else { - history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); - gb->set_input(history_2d, zeros.data(), Precision::FP16); - } - conv_source_with_history = gb->concat(history_2d, mixed_qkv, 0); - } - - const size_t total_conv_len = gb->get_output_buffer(conv_source_with_history).shape[0]; - size_t conv_input = gb->reshape(conv_source_with_history, {1, total_conv_len, mixed_proj_dim}); - size_t conv_out = gb->conv1d_causal(conv_input, layer.deltanet_conv_weight, kernel, 1); - const size_t conv_start = total_conv_len > seq_len ? total_conv_len - seq_len : 0; - conv_out = gb->slice(conv_out, 1, conv_start, seq_len); - mixed_qkv = gb->reshape(gb->silu(conv_out), {seq_len, mixed_proj_dim}); - } - - q_proj = gb->slice(mixed_qkv, 1, 0, qk_proj_dim); - k_proj = gb->slice(mixed_qkv, 1, qk_proj_dim, qk_proj_dim); - v_proj = gb->slice(mixed_qkv, 1, qk_proj_dim + qk_proj_dim, v_proj_dim); - - size_t q_4d = gb->reshape(q_proj, {1, seq_len, num_k_heads, key_dim}); - size_t k_4d = gb->reshape(k_proj, {1, seq_len, num_k_heads, key_dim}); - size_t v_4d = gb->reshape(v_proj, {1, seq_len, num_heads, value_dim}); - - size_t q_norm = gb->sum(gb->multiply(q_4d, q_4d), 3); - q_norm = gb->scalar_sqrt(gb->scalar_add(q_norm, 1e-6f)); - q_norm = gb->reshape(q_norm, {1, seq_len, num_k_heads, 1}); - q_4d = gb->divide(q_4d, q_norm); - - size_t k_norm = gb->sum(gb->multiply(k_4d, k_4d), 3); - k_norm = gb->scalar_sqrt(gb->scalar_add(k_norm, 1e-6f)); - k_norm = gb->reshape(k_norm, {1, seq_len, num_k_heads, 1}); - k_4d = gb->divide(k_4d, k_norm); - - size_t a_logits = gb->matmul(normalized_input, layer.deltanet_gate_weight, true, backend); - size_t b_logits = gb->matmul(normalized_input, layer.deltanet_beta_weight, true, backend); - a_logits = gb->precision_cast(a_logits, Precision::FP16); - b_logits = gb->precision_cast(b_logits, Precision::FP16); - if (layer.deltanet_beta_bias) { - size_t dt_bias_vec = layer.deltanet_beta_bias; - const auto& dt_buf = gb->get_output_buffer(dt_bias_vec); - if (!dt_buf.shape.empty() && dt_buf.shape[0] > num_heads) { - dt_bias_vec = gb->slice(dt_bias_vec, 0, 0, num_heads); - } - size_t dt_bias_2d = gb->reshape(dt_bias_vec, {1, num_heads}); - dt_bias_2d = gb->precision_cast(dt_bias_2d, Precision::FP16); - a_logits = gb->add(a_logits, dt_bias_2d); - } - size_t a_softplus = gb->scalar_log(gb->scalar_add(gb->scalar_exp(a_logits), 1.0f)); - - size_t gate_log; - if (layer.deltanet_gate_bias) { - size_t a_log_vec = layer.deltanet_gate_bias; - const auto& a_log_buf = gb->get_output_buffer(a_log_vec); - if (!a_log_buf.shape.empty() && a_log_buf.shape[0] > num_heads) { - a_log_vec = gb->slice(a_log_vec, 0, 0, num_heads); - } - size_t a_log_2d = gb->reshape(a_log_vec, {1, num_heads}); - a_log_2d = gb->precision_cast(a_log_2d, Precision::FP16); - size_t neg_exp_a = gb->scalar_multiply(gb->scalar_exp(a_log_2d), -1.0f); - gate_log = gb->multiply(neg_exp_a, a_softplus); - } else { - gate_log = gb->scalar_multiply(a_softplus, -1.0f); - } - size_t beta = gb->sigmoid(b_logits); - - size_t gate_3d = gb->reshape(gate_log, {1, seq_len, num_heads}); - size_t beta_3d = gb->reshape(beta, {1, seq_len, num_heads}); - - size_t initial_state; - - if (prev_state_flat != 0) { - initial_state = gb->reshape(prev_state_flat, {1, key_dim, num_heads, value_dim}); - } else { - initial_state = gb->input({1, key_dim, num_heads, value_dim}, Precision::FP16); - std::vector<__fp16> zeros(state_flat_dim, static_cast<__fp16>(0.0f)); - gb->set_input(initial_state, zeros.data(), Precision::FP16); - } - - size_t deltanet_out; - if (use_cache && seq_len == 1) { - deltanet_out = gb->gated_deltanet_decode(q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, 0.0f); - } else { - const size_t chunk_for_op = std::min(64, std::max(1, seq_len)); - deltanet_out = gb->gated_deltanet_prefill( - q_4d, k_4d, v_4d, gate_3d, beta_3d, initial_state, chunk_for_op, 0.0f); - } - - size_t y_4d = gb->slice(deltanet_out, 1, 0, seq_len); - size_t state_tail = gb->slice(deltanet_out, 1, seq_len, key_dim); - - if (use_cache) { - size_t packed_cache = gb->reshape(state_tail, {1, state_flat_dim}); - if (deltanet_conv_flat_dim_ > 0) { - size_t history_2d = 0; - if (layer.deltanet_conv_weight) { - const size_t total_history_len = gb->get_output_buffer(conv_source_with_history).shape[0]; - if (total_history_len >= deltanet_conv_history_len_) { - const size_t hist_start = total_history_len - deltanet_conv_history_len_; - history_2d = gb->slice(conv_source_with_history, 0, hist_start, deltanet_conv_history_len_); - } else { - size_t pad_rows = deltanet_conv_history_len_ - total_history_len; - size_t pad = gb->input({pad_rows, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(pad_rows * mixed_proj_dim, static_cast<__fp16>(0.0f)); - gb->set_input(pad, zeros.data(), Precision::FP16); - history_2d = gb->concat(pad, conv_source_with_history, 0); - } - } else if (prev_conv_flat != 0) { - history_2d = gb->reshape(prev_conv_flat, {deltanet_conv_history_len_, mixed_proj_dim}); - history_2d = gb->precision_cast(history_2d, Precision::FP16); - } else { - history_2d = gb->input({deltanet_conv_history_len_, mixed_proj_dim}, Precision::FP16); - std::vector<__fp16> zeros(deltanet_conv_flat_dim_, static_cast<__fp16>(0.0f)); - gb->set_input(history_2d, zeros.data(), Precision::FP16); - } - - size_t history_flat = gb->reshape(history_2d, {1, deltanet_conv_flat_dim_}); - packed_cache = gb->concat(packed_cache, history_flat, 1); - } - packed_cache = gb->precision_cast(packed_cache, conv_cache_.precision); - conv_cache_state_nodes_[layer_idx] = packed_cache; - cache_k_output_nodes_[layer_idx] = 0; - cache_v_output_nodes_[layer_idx] = 0; - } else { - conv_cache_state_nodes_[layer_idx] = 0; - } - - size_t y_2d = gb->reshape(y_4d, {seq_len * num_heads, value_dim}); - if (z_proj != 0) { - size_t z_3d = gb->reshape(z_proj, {seq_len, num_heads, value_dim}); - size_t z_2d = gb->reshape(z_3d, {seq_len * num_heads, value_dim}); - y_2d = gb->multiply(gb->rms_norm(y_2d, layer.attn_q_norm_weight, config_.layer_norm_eps), gb->silu(z_2d)); - } - y_2d = gb->reshape(y_2d, {seq_len, num_heads * value_dim}); - return gb->matmul(y_2d, layer.attn_output_weight, true, backend); -} - -size_t Qwen3p5Model::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer_entry = weight_nodes_.layers[layer_idx]; - const auto& layer = layer_entry.weights; - - if (layer_entry.type == WeightNodeIDs::LayerType::DELTANET) { - return build_gated_deltanet(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - } - - auto q_proj = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - auto k_proj = gb->matmul(normalized_input, layer.attn_k_weight, true, backend); - auto v_proj = gb->matmul(normalized_input, layer.attn_v_weight, true, backend); - - const auto& q_shape = gb->get_output_buffer(q_proj).shape; - size_t batch_seq = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - const size_t q_target_dim = num_heads * head_dim; - - size_t q_gate = 0; - const size_t q_proj_dim = q_shape.size() > 1 ? q_shape[1] : 0; - if (q_proj_dim == 2 * q_target_dim) { - size_t q_packed = gb->reshape(q_proj, {batch_seq, num_heads, 2 * head_dim}); - q_gate = gb->slice(q_packed, 2, head_dim, head_dim); - q_proj = gb->slice(q_packed, 2, 0, head_dim); - q_gate = gb->reshape(q_gate, {batch_seq, q_target_dim}); - q_proj = gb->reshape(q_proj, {batch_seq, q_target_dim}); - } - - const size_t q_norm_weight = gb->scalar_add(layer.attn_q_norm_weight, 1.0f); - q_proj = gb->reshape(q_proj, {batch_seq * num_heads, head_dim}); - q_proj = gb->rms_norm(q_proj, q_norm_weight, config_.layer_norm_eps); - q_proj = gb->reshape(q_proj, {batch_seq, num_heads * head_dim}); - - size_t num_kv_heads = config_.attention_kv_heads; - const size_t k_norm_weight = gb->scalar_add(layer.attn_k_norm_weight, 1.0f); - k_proj = gb->reshape(k_proj, {batch_seq * num_kv_heads, head_dim}); - k_proj = gb->rms_norm(k_proj, k_norm_weight, config_.layer_norm_eps); - k_proj = gb->reshape(k_proj, {batch_seq, num_kv_heads * head_dim}); - - size_t seq_len = batch_seq; - - auto q_proj_4d = gb->reshape(q_proj, {1, seq_len, config_.attention_heads, config_.attention_head_dim}); - auto k_proj_4d = gb->reshape(k_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - auto v_proj_4d = gb->reshape(v_proj, {1, seq_len, config_.attention_kv_heads, config_.attention_head_dim}); - - if (config_.rope_theta > 0) { - float rotary_factor = config_.partial_rotary_factor; - if (rotary_factor <= 0.0f) { - rotary_factor = 0.25f; - } - size_t rot_dim = static_cast(static_cast(head_dim) * rotary_factor); - rot_dim = std::min(rot_dim, head_dim); - if ((rot_dim & 1u) != 0) { - rot_dim -= 1; - } - - auto apply_partial_rope = [&](size_t x_4d) -> size_t { - if (rot_dim == 0) { - return x_4d; - } - if (rot_dim >= head_dim) { - return gb->rope(x_4d, config_.rope_theta, position_offset); - } - size_t x_rot = gb->slice(x_4d, 3, 0, rot_dim); - size_t x_tail = gb->slice(x_4d, 3, rot_dim, head_dim - rot_dim); - x_rot = gb->rope(x_rot, config_.rope_theta, position_offset); - return gb->concat(x_rot, x_tail, 3); - }; - - q_proj_4d = apply_partial_rope(q_proj_4d); - k_proj_4d = apply_partial_rope(k_proj_4d); - } - - size_t attn_output_4d; - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = k_proj_4d; - cache_v_output_nodes_[layer_idx] = v_proj_4d; - conv_cache_state_nodes_[layer_idx] = 0; - } - - if (use_cache && !kv_cache_.is_empty()) { - attn_output_4d = gb->attention_int8_hybrid( - q_proj_4d, k_proj_4d, v_proj_4d, - attention_scale_, position_offset, - kv_cache_.get_keys_int8(layer_idx), - kv_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - kv_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, head_dim - ); - } else { - attn_output_4d = gb->attention(q_proj_4d, k_proj_4d, v_proj_4d, attention_scale_, position_offset); - } - - auto attn_output = gb->reshape(attn_output_4d, {seq_len, config_.attention_head_dim * config_.attention_heads}); - if (q_gate != 0) { - attn_output = gb->multiply(attn_output, gb->sigmoid(q_gate)); - } - return gb->matmul(attn_output, layer.attn_output_weight, true, backend); -} - -size_t Qwen3p5Model::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx].weights; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - -size_t Qwen3p5Model::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx].weights; - const size_t input_norm_weight = gb->scalar_add(layer.input_layernorm_weight, 1.0f); - auto normalized_input = gb->rms_norm(hidden, input_norm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add(hidden, attn_output); - const size_t post_attn_norm_weight = gb->scalar_add(layer.post_attention_layernorm_weight, 1.0f); - auto normalized_after_attention = gb->rms_norm(after_attention, post_attn_norm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add(after_attention, mlp_output); -} - -size_t Qwen3p5Model::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - if (conv_cache_state_nodes_.size() != config_.num_layers) { - conv_cache_state_nodes_.assign(config_.num_layers, 0); - } - std::fill(conv_cache_state_nodes_.begin(), conv_cache_state_nodes_.end(), 0); - last_forward_used_cache_ = use_cache; - if (!use_cache && conv_cache_.window_size > 0) { - conv_cache_.reset(); - deltanet_total_seq_len_ = 0; - } - - auto seq_len = static_cast(tokens.size()); - const size_t kv_pos = use_cache ? kv_cache_.get_total_seq_len() : 0; - const size_t position_offset = use_cache ? std::max(kv_pos, deltanet_total_seq_len_) : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - static std::set skip_layers = {}; - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - if (skip_layers.count(layer_idx)) { - continue; - } - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - const size_t final_norm_weight = gb->scalar_add(weight_nodes_.output_norm_weight, 1.0f); - auto final_hidden = gb->rms_norm(hidden, final_norm_weight, config_.layer_norm_eps); - return final_hidden; -} - -} -} diff --git a/cactus/models/model_siglip2.cpp b/cactus/models/model_siglip2.cpp deleted file mode 100644 index 8977c4c65..000000000 --- a/cactus/models/model_siglip2.cpp +++ /dev/null @@ -1,442 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include - -namespace cactus { -namespace engine { - -namespace { - -size_t infer_npu_patch_capacity(const std::vector& input_shape, size_t hidden_dim) { - if (input_shape.size() < 2) { - return 0; - } - - const int first = input_shape[0]; - const int second = input_shape[1]; - if (first <= 0 || second <= 0) { - return 0; - } - - if (static_cast(second) == hidden_dim) { - return static_cast(first); - } - if (static_cast(first) == hidden_dim) { - return static_cast(second); - } - - return static_cast(first); -} - -} // namespace - -Siglip2VisionModel::Siglip2VisionModel() : Model() { - config_.model_type = Config::ModelType::SIGLIP2; -} - -Siglip2VisionModel::Siglip2VisionModel(const Config& cfg) : Model(cfg) { - Siglip2Preprocessor::Config preprocessor_config; - preprocessor_config.patch_size = static_cast(config_.vision_patch_size); - preprocessor_config.downsample_factor = static_cast(config_.downsample_factor); - preprocessor_config.min_tiles = static_cast(config_.min_tiles); - preprocessor_config.max_tiles = static_cast(config_.max_tiles); - preprocessor_config.use_thumbnail = config_.use_thumbnail; - preprocessor_config.min_image_tokens = static_cast(config_.min_image_tokens); - preprocessor_config.max_image_tokens = static_cast(config_.max_image_tokens); - preprocessor_config.max_num_patches = static_cast(config_.max_num_patches); - preprocessor_config.tile_size = static_cast(config_.tile_size); - preprocessor_config.max_pixels_tolerance = config_.max_pixels_tolerance; - preprocessor_config.do_resize = true; - preprocessor_config.do_rescale = true; - preprocessor_config.do_normalize = true; - preprocessor_config.do_convert_rgb = true; - preprocessor_config.do_image_splitting = config_.do_image_splitting; - preprocessor_config.rescale_factor = config_.rescale_factor; - preprocessor_config.image_mean[0] = config_.image_mean; - preprocessor_config.image_mean[1] = config_.image_mean; - preprocessor_config.image_mean[2] = config_.image_mean; - preprocessor_config.image_std[0] = config_.image_std; - preprocessor_config.image_std[1] = config_.image_std; - preprocessor_config.image_std[2] = config_.image_std; - - preprocessor_ = Siglip2Preprocessor(preprocessor_config); -} - -void Siglip2VisionModel::ensure_cpu_vision_weights_loaded(CactusGraph* gb) { - if (cpu_vision_weights_loaded_) { - return; - } - - std::string base = model_folder_path_ + "/"; - vision_weight_nodes_.vision_layers.resize(config_.vision_num_layers); - - vision_weight_nodes_.post_layernorm_weight = gb->mmap_weights(base + "vision_post_layernorm.weights"); - vision_weight_nodes_.post_layernorm_bias = gb->mmap_weights(base + "vision_post_layernorm.bias.weights"); - - for (uint32_t i = 0; i < vision_weight_nodes_.vision_layers.size(); ++i) { - auto& layer = vision_weight_nodes_.vision_layers[i]; - std::string prefix = base + "vision_layer_" + std::to_string(i) + "_"; - - layer.attn_q_weight = gb->mmap_weights(prefix + "self_attn_q.weights"); - layer.attn_q_bias = gb->mmap_weights(prefix + "self_attn_q.bias.weights"); - layer.attn_k_weight = gb->mmap_weights(prefix + "self_attn_k.weights"); - layer.attn_k_bias = gb->mmap_weights(prefix + "self_attn_k.bias.weights"); - layer.attn_v_weight = gb->mmap_weights(prefix + "self_attn_v.weights"); - layer.attn_v_bias = gb->mmap_weights(prefix + "self_attn_v.bias.weights"); - layer.attn_output_weight = gb->mmap_weights(prefix + "self_attn_out.weights"); - layer.attn_output_bias = gb->mmap_weights(prefix + "self_attn_out.bias.weights"); - - layer.layer_norm1_weight = gb->mmap_weights(prefix + "layer_norm1.weights"); - layer.layer_norm1_bias = gb->mmap_weights(prefix + "layer_norm1.bias.weights"); - layer.layer_norm2_weight = gb->mmap_weights(prefix + "layer_norm2.weights"); - layer.layer_norm2_bias = gb->mmap_weights(prefix + "layer_norm2.bias.weights"); - - layer.mlp_fc1_weight = gb->mmap_weights(prefix + "ffn_fc1.weights"); - layer.mlp_fc1_bias = gb->mmap_weights(prefix + "ffn_fc1.bias.weights"); - layer.mlp_fc2_weight = gb->mmap_weights(prefix + "ffn_fc2.weights"); - layer.mlp_fc2_bias = gb->mmap_weights(prefix + "ffn_fc2.bias.weights"); - } - - cpu_vision_weights_loaded_ = true; -} - -void Siglip2VisionModel::load_weights_to_graph(CactusGraph* gb) { - std::string base = model_folder_path_ + "/"; - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - - std::vector input_shape = npu_encoder_->get_input_shape(); - if (input_shape.empty()) { - input_shape = { - static_cast(config_.max_num_patches), - static_cast(config_.vision_embed_dim) - }; - } - npu_encoder_->preallocate(input_shape, "x", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - // Always load patch embedding and position embedding weights (needed for both CPU and NPU paths) - vision_weight_nodes_.patch_embedding_weight = gb->mmap_weights(base + "vision_patch_embedding.weights"); - vision_weight_nodes_.patch_embedding_bias = gb->mmap_weights(base + "vision_patch_embedding.bias.weights"); - vision_weight_nodes_.position_embedding = gb->mmap_weights(base + "vision_position_embedding.weights"); - - if (!use_npu_encoder_) { - ensure_cpu_vision_weights_loaded(gb); - } -} - -Siglip2VisionModel::VisionEmbeddingResult Siglip2VisionModel::build_vision_embeddings( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - const int num_tiles = preprocessed_image.num_tiles; - const int max_patches = preprocessed_image.max_patches_per_tile; - const int patch_dim = preprocessed_image.patch_dim; - - const size_t expected_size = static_cast(num_tiles) * static_cast(max_patches) * - static_cast(patch_dim); - if (preprocessed_image.pixel_values.size() != expected_size) { - throw std::runtime_error( - "Pixel values size mismatch: expected " + std::to_string(expected_size) + - " (tiles=" + std::to_string(num_tiles) + " * max_patches=" + std::to_string(max_patches) + - " * patch_dim=" + std::to_string(patch_dim) + ") but got " + - std::to_string(preprocessed_image.pixel_values.size())); - } - for (size_t i = 0; i < std::min(100, preprocessed_image.pixel_values.size()); ++i) { - float val = preprocessed_image.pixel_values[i]; - if (std::isnan(val) || std::isinf(val)) { - throw std::runtime_error( - "Invalid value in pixel_values at index " + std::to_string(i) + ": " + std::to_string(val)); - } - } - - size_t reshaped_weight = gb->reshape( - vision_weight_nodes_.patch_embedding_weight, - {static_cast(config_.vision_embed_dim), static_cast(patch_dim)}); - - size_t patch_bias = vision_weight_nodes_.patch_embedding_bias; - std::vector tile_embeddings; - tile_embeddings.reserve(static_cast(num_tiles)); - - for (int tile_idx = 0; tile_idx < num_tiles; ++tile_idx) { - const auto& shape = preprocessed_image.spatial_shapes[tile_idx]; - const int tile_h = shape.first; - const int tile_w = shape.second; - const int actual_patches = tile_h * tile_w; - if (actual_patches <= 0) { - continue; - } - - const float* tile_data = preprocessed_image.pixel_values.data() + - static_cast(tile_idx) * static_cast(max_patches) * - static_cast(patch_dim); - - size_t tile_input_fp32 = gb->input( - {static_cast(actual_patches), static_cast(patch_dim)}, Precision::FP32); - gb->set_input(tile_input_fp32, tile_data, Precision::FP32); - size_t tile_input = gb->precision_cast(tile_input_fp32, Precision::FP16); - size_t tile_patch = gb->matmul(tile_input, reshaped_weight, true, backend); - size_t tile_bias = gb->add(tile_patch, patch_bias); - size_t tile_pos = gb->bilinear_interpolation( - vision_weight_nodes_.position_embedding, - static_cast(tile_h), - static_cast(tile_w), - /*align_corners=*/false); - size_t tile_pos_cast = gb->precision_cast(tile_pos, Precision::FP16); - size_t tile_embed = gb->add(tile_bias, tile_pos_cast); - tile_embeddings.push_back(tile_embed); - } - - if (tile_embeddings.empty()) { - throw std::runtime_error("No valid tiles produced embeddings in build_vision_embeddings"); - } - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - size_t embeddings = concat_nodes(tile_embeddings); - return VisionEmbeddingResult{embeddings, std::move(tile_embeddings)}; -} - -size_t Siglip2VisionModel::build_vision_attention(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t q = gb->matmul(hidden_states, layer.attn_q_weight, true, backend); - q = gb->add(q, layer.attn_q_bias); - size_t k = gb->matmul(hidden_states, layer.attn_k_weight, true, backend); - k = gb->add(k, layer.attn_k_bias); - size_t v = gb->matmul(hidden_states, layer.attn_v_weight, true, backend); - v = gb->add(v, layer.attn_v_bias); - const size_t num_heads = static_cast(config_.vision_attention_heads); - const size_t head_dim = static_cast(config_.vision_embed_dim / config_.vision_attention_heads); - const auto& q_buf = gb->get_output_buffer(q); - size_t seq_len = q_buf.shape[0]; - - size_t q_4d = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - size_t k_4d = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - size_t v_4d = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - float scale = 1.0f / std::sqrt(static_cast(head_dim)); - size_t attn_output = gb->attention(q_4d, k_4d, v_4d, scale, false, backend); - size_t attn_2d = gb->reshape(attn_output, {seq_len, num_heads * head_dim}); - size_t output = gb->matmul(attn_2d, layer.attn_output_weight, true, backend); - output = gb->add(output, layer.attn_output_bias); - return output; -} - -size_t Siglip2VisionModel::build_vision_mlp(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t fc1_output = gb->matmul(hidden_states, layer.mlp_fc1_weight, true, backend); - fc1_output = gb->add(fc1_output, layer.mlp_fc1_bias); - size_t activated = gb->gelu(fc1_output); - size_t fc2_output = gb->matmul(activated, layer.mlp_fc2_weight, true, backend); - fc2_output = gb->add(fc2_output, layer.mlp_fc2_bias); - return fc2_output; -} - -size_t Siglip2VisionModel::build_vision_transformer_layer(CactusGraph* gb, size_t hidden_states, - uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = vision_weight_nodes_.vision_layers[layer_idx]; - - size_t residual = hidden_states; - size_t normalized = gb->layernorm(hidden_states, layer.layer_norm1_weight, - layer.layer_norm1_bias, config_.layer_norm_eps); - size_t attn_output = build_vision_attention(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, attn_output); - residual = hidden_states; - normalized = gb->layernorm(hidden_states, layer.layer_norm2_weight, - layer.layer_norm2_bias, config_.layer_norm_eps); - size_t mlp_output = build_vision_mlp(gb, normalized, layer_idx, backend); - hidden_states = gb->add(residual, mlp_output); - return hidden_states; -} - -size_t Siglip2VisionModel::forward_vision( - CactusGraph* gb, - const Siglip2Preprocessor::PreprocessedImage& preprocessed_image, - ComputeBackend backend) { - - bool can_use_npu_encoder = use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available(); - size_t total_patches = 0; - for (const auto& shape : preprocessed_image.spatial_shapes) { - total_patches += shape.first * shape.second; - } - - if (can_use_npu_encoder) { - size_t max_npu_patches = - infer_npu_patch_capacity(npu_encoder_->get_input_shape(), config_.vision_embed_dim); - if (max_npu_patches > 0 && total_patches > max_npu_patches) { - CACTUS_LOG_WARN("npu", - "[siglip2-vision] image has " - << total_patches << " patches but the NPU encoder supports " - << max_npu_patches << "; falling back to CPU vision encoder"); - can_use_npu_encoder = false; - } - } - - if (can_use_npu_encoder) { - // NPU path: build patch embeddings + position embeddings on CPU, then run transformer on NPU - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - gb->execute(); - - const auto& embed_buffer = gb->get_output_buffer(embedding_result.combined_embeddings); - void* embed_ptr = gb->get_output(embedding_result.combined_embeddings); - - std::vector<__fp16> embed_f16; - const __fp16* input_ptr; - if (embed_buffer.precision == Precision::FP16) { - input_ptr = static_cast(embed_ptr); - } else { - embed_f16.resize(total_patches * config_.vision_embed_dim); - cactus_fp32_to_fp16(static_cast(embed_ptr), - embed_f16.data(), embed_f16.size()); - input_ptr = embed_f16.data(); - } - - std::vector input_shape = { - static_cast(total_patches), - static_cast(config_.vision_embed_dim) - }; - - __fp16* output_buffer = npu_encoder_->get_output_buffer(); - if (output_buffer) { - size_t elements = npu_encoder_->encode( - input_ptr, output_buffer, input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_external_input(vision_output, output_buffer, Precision::FP16); - return vision_output; - } - } else { - std::vector<__fp16> npu_output(total_patches * config_.vision_embed_dim); - size_t elements = npu_encoder_->encode( - input_ptr, npu_output.data(), input_shape, "x", ""); - - if (elements > 0) { - gb->soft_reset(); - size_t vision_output = gb->input({total_patches, config_.vision_embed_dim}, - Precision::FP16); - gb->set_input(vision_output, npu_output.data(), Precision::FP16); - return vision_output; - } - } - - CACTUS_LOG_WARN("npu", "[siglip2-vision] NPU encoder failed; falling back to CPU vision encoder"); - gb->soft_reset(); - } - - ensure_cpu_vision_weights_loaded(gb); - - // CPU path: full forward pass through transformer layers - auto embedding_result = build_vision_embeddings(gb, preprocessed_image, backend); - - auto concat_nodes = [&](const std::vector& nodes) { - if (nodes.empty()) { - throw std::runtime_error("Attempted to concatenate an empty node list in forward_vision"); - } - size_t combined = nodes.front(); - for (size_t i = 1; i < nodes.size(); ++i) { - combined = gb->concat(combined, nodes[i], /*axis=*/0); - } - return combined; - }; - - std::vector tile_outputs; - tile_outputs.reserve(embedding_result.tile_embeddings.size()); - - for (size_t tile_idx = 0; tile_idx < embedding_result.tile_embeddings.size(); ++tile_idx) { - size_t hidden_states = embedding_result.tile_embeddings[tile_idx]; - - for (uint32_t layer_idx = 0; layer_idx < config_.vision_num_layers; ++layer_idx) { - hidden_states = build_vision_transformer_layer(gb, hidden_states, layer_idx, backend); - } - - hidden_states = gb->layernorm(hidden_states, - vision_weight_nodes_.post_layernorm_weight, - vision_weight_nodes_.post_layernorm_bias, - config_.layer_norm_eps); - tile_outputs.push_back(hidden_states); - } - - if (tile_outputs.empty()) { - throw std::runtime_error("No tile outputs generated in forward_vision"); - } - - size_t combined_output = concat_nodes(tile_outputs); - - return combined_output; -} - -size_t Siglip2VisionModel::forward_vision(const Siglip2Preprocessor::PreprocessedImage& preprocessed_image) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - auto backend = config_.default_backend == Config::Backend::CPU ? ComputeBackend::CPU : ComputeBackend::NPU; - return forward_vision(gb, preprocessed_image, backend); -} - -std::vector Siglip2VisionModel::get_image_embedding(const std::string& image_path) { - auto preprocessed = preprocessor_.preprocess_from_file(image_path); - size_t last_hidden_state = forward_vision(preprocessed); - - auto* gb = static_cast(graph_handle_); - size_t pooled = gb->mean(last_hidden_state, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - return embedding; -} - -size_t Siglip2VisionModel::forward(const std::vector&, bool) {return 0;} - -size_t Siglip2VisionModel::build_attention(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -size_t Siglip2VisionModel::build_mlp(CactusGraph*, size_t, uint32_t, - ComputeBackend) const { - return 0; -} - -size_t Siglip2VisionModel::build_transformer_block(CactusGraph*, size_t, uint32_t, - ComputeBackend, bool, size_t) { - return 0; -} - -} -} diff --git a/cactus/models/model_silero_vad.cpp b/cactus/models/model_silero_vad.cpp deleted file mode 100644 index 7573ac2de..000000000 --- a/cactus/models/model_silero_vad.cpp +++ /dev/null @@ -1,338 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include -#include - -namespace cactus { -namespace engine { - -SileroVADModel::SileroVADModel() : Model() { - weight_nodes_.encoder_blocks.resize(4); -} - -SileroVADModel::SileroVADModel(const Config& config) : Model(config) { - weight_nodes_.encoder_blocks.resize(4); -} - -SileroVADModel::~SileroVADModel() = default; - -void SileroVADModel::load_weights_to_graph(CactusGraph* gb) { - weight_nodes_.stft_basis = gb->mmap_weights(model_folder_path_ + "/stft_basis.weights"); - - for (uint32_t i = 0; i < 4; i++) { - std::string block_prefix = model_folder_path_ + "/encoder_block_" + std::to_string(i) + "_"; - weight_nodes_.encoder_blocks[i].conv_weight = gb->mmap_weights(block_prefix + "conv_weight.weights"); - weight_nodes_.encoder_blocks[i].conv_bias = gb->mmap_weights(block_prefix + "conv_bias.weights"); - } - - weight_nodes_.lstm_weight_ih = gb->mmap_weights(model_folder_path_ + "/lstm_weight_ih.weights"); - weight_nodes_.lstm_weight_hh = gb->mmap_weights(model_folder_path_ + "/lstm_weight_hh.weights"); - weight_nodes_.lstm_bias_ih = gb->mmap_weights(model_folder_path_ + "/lstm_bias_ih.weights"); - weight_nodes_.lstm_bias_hh = gb->mmap_weights(model_folder_path_ + "/lstm_bias_hh.weights"); - - weight_nodes_.output_conv_weight = gb->mmap_weights(model_folder_path_ + "/output_conv_weight.weights"); - weight_nodes_.output_conv_bias = gb->mmap_weights(model_folder_path_ + "/output_conv_bias.weights"); -} - -void SileroVADModel::build_graph() { - const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; - - graph_nodes_.input = graph_.input({1, 1, input_size}, Precision::FP16); - graph_nodes_.h_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); - graph_nodes_.c_prev = graph_.input({1, HIDDEN_SIZE}, Precision::FP16); - - auto cplx = graph_.stft(graph_nodes_.input, weight_nodes_.stft_basis, 128, 129); - auto re = graph_.slice(cplx, 1, 0, 129); - auto im = graph_.slice(cplx, 1, 129, 129); - auto re_sq = graph_.multiply(re, re); - auto im_sq = graph_.multiply(im, im); - auto mag = graph_.scalar_sqrt(graph_.add(re_sq, im_sq)); - - const size_t strides[4] = {1, 2, 2, 1}; - auto x = mag; - for (uint32_t i = 0; i < 4; i++) { - auto conv = graph_.conv1d_k3(x, weight_nodes_.encoder_blocks[i].conv_weight, strides[i]); - auto bias_reshaped = graph_.reshape(weight_nodes_.encoder_blocks[i].conv_bias, - {1, graph_.get_output_buffer(weight_nodes_.encoder_blocks[i].conv_bias).shape[0], 1}); - auto with_bias = graph_.add(conv, bias_reshaped); - x = graph_.relu(with_bias); - } - - auto x_squeezed = graph_.reshape(x, {1, HIDDEN_SIZE}); - - graph_nodes_.lstm_output = graph_.lstm_cell( - x_squeezed, - graph_nodes_.h_prev, - graph_nodes_.c_prev, - weight_nodes_.lstm_weight_ih, - weight_nodes_.lstm_weight_hh, - weight_nodes_.lstm_bias_ih, - weight_nodes_.lstm_bias_hh - ); - - auto h_new = graph_.slice(graph_nodes_.lstm_output, 2, 0, 1); - auto c_new = graph_.slice(graph_nodes_.lstm_output, 2, 1, 1); - graph_nodes_.h_new = graph_.reshape(h_new, {1, HIDDEN_SIZE}); - graph_nodes_.c_new = graph_.reshape(c_new, {1, HIDDEN_SIZE}); - - auto h_relu = graph_.relu(graph_nodes_.h_new); - auto h_unsqueezed = graph_.reshape(h_relu, {1, HIDDEN_SIZE, 1}); - auto logits = graph_.conv1d(h_unsqueezed, weight_nodes_.output_conv_weight, weight_nodes_.output_conv_bias, 1); - graph_nodes_.output = graph_.sigmoid(logits); -} - -bool SileroVADModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - state_.h.resize(HIDDEN_SIZE, (__fp16)0.0f); - state_.c.resize(HIDDEN_SIZE, (__fp16)0.0f); - state_.context.resize(CONTEXT_SIZE, 0.0f); - state_.input_buf.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE, 0.0f); - state_.input_fp16.resize(CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE); - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - build_graph(); - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -float SileroVADModel::process_chunk(const std::vector& audio_chunk) { - const float* audio = audio_chunk.data(); - size_t samples = audio_chunk.size(); - - if (samples != CHUNK_SIZE) { - throw std::runtime_error("SileroVAD: Expected 512 samples"); - } - if (!initialized_) { - throw std::runtime_error("SileroVAD: Not initialized"); - } - - const size_t input_size = CONTEXT_SIZE + CHUNK_SIZE + REFLECT_PAD_SIZE; - - float* input_buf = state_.input_buf.data(); - __fp16* input_fp16 = state_.input_fp16.data(); - - std::memcpy(input_buf, state_.context.data(), CONTEXT_SIZE * sizeof(float)); - std::memcpy(input_buf + CONTEXT_SIZE, audio, CHUNK_SIZE * sizeof(float)); - - for (size_t i = 0; i < REFLECT_PAD_SIZE; i++) { - input_buf[CONTEXT_SIZE + CHUNK_SIZE + i] = - input_buf[CONTEXT_SIZE + CHUNK_SIZE - 2 - i]; - } - - cactus_fp32_to_fp16(input_buf, input_fp16, input_size); - - graph_.set_input(graph_nodes_.input, input_fp16, Precision::FP16); - graph_.set_input(graph_nodes_.h_prev, state_.h.data(), Precision::FP16); - graph_.set_input(graph_nodes_.c_prev, state_.c.data(), Precision::FP16); - graph_.execute(); - - void* h_out = graph_.get_output(graph_nodes_.h_new); - void* c_out = graph_.get_output(graph_nodes_.c_new); - void* vad_out = graph_.get_output(graph_nodes_.output); - - const __fp16* h_new = static_cast(h_out); - const __fp16* c_new = static_cast(c_out); - const __fp16* vad_score = static_cast(vad_out); - - std::memcpy(state_.h.data(), h_new, HIDDEN_SIZE * sizeof(__fp16)); - std::memcpy(state_.c.data(), c_new, HIDDEN_SIZE * sizeof(__fp16)); - - float vad_score_f32 = static_cast(vad_score[0]); - - std::memcpy(state_.context.data(), input_buf + CHUNK_SIZE, CONTEXT_SIZE * sizeof(float)); - - return vad_score_f32; -} - -void SileroVADModel::reset_states() { - state_.h.assign(HIDDEN_SIZE, (__fp16)0.0f); - state_.c.assign(HIDDEN_SIZE, (__fp16)0.0f); - state_.context.assign(CONTEXT_SIZE, 0.0f); -} - -std::vector SileroVADModel::get_speech_timestamps(const std::vector& audio, const SpeechTimestampsOptions& options) { - const size_t audio_length_samples = audio.size(); - const size_t window_size_samples = options.window_size_samples; - const float min_speech_samples = options.sampling_rate * options.min_speech_duration_ms / 1000.0f; - const float speech_pad_samples = options.sampling_rate * options.speech_pad_ms / 1000.0f; - const float max_speech_samples = options.sampling_rate * options.max_speech_duration_s - window_size_samples - 2 * speech_pad_samples; - const float min_silence_samples = options.sampling_rate * options.min_silence_duration_ms / 1000.0f; - const float neg_threshold = (options.neg_threshold == 0.0f) - ? std::max(options.threshold - 0.15f, 0.01f) - : options.neg_threshold; - reset_states(); - - std::vector speech_probs; - speech_probs.reserve(audio_length_samples / window_size_samples + 1); - - std::vector chunk(window_size_samples, 0.0f); - - for (size_t current_start = 0; current_start < audio_length_samples; current_start += window_size_samples) { - size_t remaining = audio_length_samples - current_start; - size_t copy_len = std::min(remaining, window_size_samples); - - std::memcpy(chunk.data(), audio.data() + current_start, copy_len * sizeof(float)); - - if (copy_len < window_size_samples) { - std::memset(chunk.data() + copy_len, 0, (window_size_samples - copy_len) * sizeof(float)); - } - - float speech_prob = process_chunk(chunk); - speech_probs.push_back(speech_prob); - } - - bool triggered = false; - std::vector speeches; - SpeechTimestamp current_speech = {0, 0}; - size_t temp_end = 0; - size_t prev_end = 0; - size_t next_start = 0; - std::vector> possible_ends; - - const float min_silence_samples_at_max_speech = options.sampling_rate * options.min_silence_at_max_speech / 1000.0f; - - for (size_t i = 0; i < speech_probs.size(); ++i) { - float speech_prob = speech_probs[i]; - size_t cur_sample = window_size_samples * i; - - if (speech_prob >= options.threshold && temp_end) { - size_t sil_dur = cur_sample - temp_end; - if (sil_dur > min_silence_samples_at_max_speech) { - possible_ends.push_back({temp_end, sil_dur}); - } - temp_end = 0; - if (next_start < prev_end) { - next_start = cur_sample; - } - } - - if (speech_prob >= options.threshold && !triggered) { - triggered = true; - current_speech.start = cur_sample; - continue; - } - - if (triggered && (cur_sample - current_speech.start > max_speech_samples)) { - if (options.use_max_poss_sil_at_max_speech && !possible_ends.empty()) { - auto max_silence = std::max_element(possible_ends.begin(), possible_ends.end(), - [](const std::pair& a, const std::pair& b) { - return a.second < b.second; - }); - prev_end = max_silence->first; - size_t dur = max_silence->second; - current_speech.end = prev_end; - speeches.push_back(current_speech); - current_speech = {0, 0}; - next_start = prev_end + dur; - - if (next_start < prev_end + cur_sample) { - current_speech.start = next_start; - } else { - triggered = false; - } - prev_end = next_start = temp_end = 0; - possible_ends.clear(); - } else { - if (prev_end) { - current_speech.end = prev_end; - speeches.push_back(current_speech); - current_speech = {0, 0}; - if (next_start < prev_end) { - triggered = false; - } else { - current_speech.start = next_start; - } - prev_end = next_start = temp_end = 0; - possible_ends.clear(); - } else { - current_speech.end = cur_sample; - speeches.push_back(current_speech); - current_speech = {0, 0}; - prev_end = next_start = temp_end = 0; - triggered = false; - possible_ends.clear(); - continue; - } - } - } - - if (speech_prob < neg_threshold && triggered) { - if (!temp_end) { - temp_end = cur_sample; - } - size_t sil_dur_now = cur_sample - temp_end; - - if (!options.use_max_poss_sil_at_max_speech && sil_dur_now > min_silence_samples_at_max_speech) { - prev_end = temp_end; - } - - if (sil_dur_now < min_silence_samples) { - continue; - } else { - current_speech.end = temp_end; - if ((current_speech.end - current_speech.start) > min_speech_samples) { - speeches.push_back(current_speech); - } - current_speech = {0, 0}; - prev_end = next_start = temp_end = 0; - triggered = false; - possible_ends.clear(); - continue; - } - } - } - - if (triggered && (audio_length_samples - current_speech.start) > min_speech_samples) { - current_speech.end = audio_length_samples; - speeches.push_back(current_speech); - } - - for (size_t i = 0; i < speeches.size(); ++i) { - if (i == 0) { - speeches[i].start = speeches[i].start > static_cast(speech_pad_samples) - ? speeches[i].start - static_cast(speech_pad_samples) - : 0; - } - if (i != speeches.size() - 1) { - size_t silence_duration = speeches[i + 1].start - speeches[i].end; - if (silence_duration < 2 * static_cast(speech_pad_samples)) { - speeches[i].end += silence_duration / 2; - speeches[i + 1].start = speeches[i + 1].start > silence_duration / 2 - ? speeches[i + 1].start - silence_duration / 2 - : 0; - } else { - speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); - speeches[i + 1].start = speeches[i + 1].start > static_cast(speech_pad_samples) - ? speeches[i + 1].start - static_cast(speech_pad_samples) - : 0; - } - } else { - speeches[i].end = std::min(audio_length_samples, speeches[i].end + static_cast(speech_pad_samples)); - } - } - - return speeches; -} - -size_t SileroVADModel::forward(const std::vector&, const std::vector&, bool) { - return 0; -} - -} -} diff --git a/cactus/models/model_wespeaker.cpp b/cactus/models/model_wespeaker.cpp deleted file mode 100644 index 62d4642c8..000000000 --- a/cactus/models/model_wespeaker.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../kernel/kernel.h" -#include - -namespace cactus { -namespace engine { - -WeSpeakerModel::WeSpeakerModel() : Model() {} -WeSpeakerModel::WeSpeakerModel(const Config& config) : Model(config) {} - -WeSpeakerModel::ResBlockWeights WeSpeakerModel::load_resblock(CactusGraph* gb, const std::string& prefix, bool has_shortcut) { - ResBlockWeights rb; - rb.has_shortcut = has_shortcut; - rb.conv1_w = gb->mmap_weights(prefix + "_conv1_weight.weights"); - rb.conv1_b = gb->mmap_weights(prefix + "_conv1_bias.weights"); - rb.conv2_w = gb->mmap_weights(prefix + "_conv2_weight.weights"); - rb.conv2_b = gb->mmap_weights(prefix + "_conv2_bias.weights"); - if (has_shortcut) { - rb.shortcut_conv_w = gb->mmap_weights(prefix + "_shortcut_0_weight.weights"); - rb.shortcut_conv_b = gb->mmap_weights(prefix + "_shortcut_0_bias.weights"); - } - return rb; -} - -size_t WeSpeakerModel::build_resblock(CactusGraph* gb, size_t x, const ResBlockWeights& rb, bool stride2) { - size_t identity = x; - - size_t out = stride2 ? gb->conv2d_k3s2p1(x, rb.conv1_w, rb.conv1_b) - : gb->conv2d_k3s1p1(x, rb.conv1_w, rb.conv1_b); - out = gb->relu(out); - out = gb->conv2d_k3s1p1(out, rb.conv2_w, rb.conv2_b); - - if (rb.has_shortcut) { - identity = gb->conv2d_k3s2p1(x, rb.shortcut_conv_w, rb.shortcut_conv_b); - } - - out = gb->add(out, identity); - out = gb->relu(out); - return out; -} - -void WeSpeakerModel::load_weights_to_graph(CactusGraph* gb) { - const std::string& p = model_folder_path_; - - weight_nodes_.conv1_w = gb->mmap_weights(p + "/resnet_conv1_weight.weights"); - weight_nodes_.conv1_b = gb->mmap_weights(p + "/resnet_conv1_bias.weights"); - - auto load_layer = [&](const std::string& layer_name, int num_blocks, bool first_has_shortcut) { - std::vector blocks; - for (int i = 0; i < num_blocks; ++i) { - std::string prefix = p + "/resnet_" + layer_name + "_" + std::to_string(i); - blocks.push_back(load_resblock(gb, prefix, i == 0 && first_has_shortcut)); - } - return blocks; - }; - - weight_nodes_.layer1 = load_layer("layer1", 3, false); - weight_nodes_.layer2 = load_layer("layer2", 4, true); - weight_nodes_.layer3 = load_layer("layer3", 6, true); - weight_nodes_.layer4 = load_layer("layer4", 3, true); - - weight_nodes_.seg1_w = gb->mmap_weights(p + "/resnet_seg_1_weight.weights"); - weight_nodes_.seg1_b = gb->mmap_weights(p + "/resnet_seg_1_bias.weights"); -} - -size_t WeSpeakerModel::build_resnet_body(CactusGraph* gb, size_t x) { - x = gb->conv2d_k3s1p1(x, weight_nodes_.conv1_w, weight_nodes_.conv1_b); - x = gb->relu(x); - - for (auto& rb : weight_nodes_.layer1) x = build_resblock(gb, x, rb, false); - for (size_t i = 0; i < weight_nodes_.layer2.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer2[i], i == 0); - for (size_t i = 0; i < weight_nodes_.layer3.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer3[i], i == 0); - for (size_t i = 0; i < weight_nodes_.layer4.size(); ++i) x = build_resblock(gb, x, weight_nodes_.layer4[i], i == 0); - return x; -} - -void WeSpeakerModel::build_graph(size_t num_frames) { - audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); - - size_t x = build_resnet_body(&graph_, audio_input_); - - x = graph_.stats_pool(x); - x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); - - output_node_ = x; -} - -void WeSpeakerModel::build_graph_masked(size_t num_frames) { - audio_input_ = graph_.input({1, 1, 80, num_frames}, Precision::FP16); - - size_t x = build_resnet_body(&graph_, audio_input_); - - const auto& post_resnet = graph_.get_output_buffer(x); - size_t T_post = post_resnet.shape.back(); - - mask_input_ = graph_.input({1, T_post}, Precision::FP32); - - x = graph_.weighted_stats_pool(x, mask_input_); - x = graph_.add(graph_.matmul(x, weight_nodes_.seg1_w, true), weight_nodes_.seg1_b); - - output_node_ = x; -} - -bool WeSpeakerModel::init(const std::string& model_folder, size_t context_size, - const std::string& system_prompt, bool do_warmup) { - (void)context_size; - (void)system_prompt; - (void)do_warmup; - - if (initialized_) { - return true; - } - - model_folder_path_ = model_folder; - - try { - load_weights_to_graph(&graph_); - initialized_ = true; - return true; - } catch (const std::exception& e) { - initialized_ = false; - return false; - } -} - -std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features) { - if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); - - static constexpr size_t NUM_MEL = 80; - const size_t num_frames = num_features / NUM_MEL; - if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); - - if (num_frames != current_num_frames_ || current_masked_) { - graph_.hard_reset(); - load_weights_to_graph(&graph_); - build_graph(num_frames); - current_num_frames_ = num_frames; - current_masked_ = false; - } - - input_buf_.resize(num_features); - for (size_t i = 0; i < num_features; ++i) - input_buf_[i] = static_cast<__fp16>(fbank_features[i]); - - graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); - graph_.execute(); - - const auto& out_buf = graph_.get_output_buffer(output_node_); - const __fp16* out_data = out_buf.data_as<__fp16>(); - size_t total = out_buf.total_size; - - std::vector result(total); - for (size_t i = 0; i < total; ++i) - result[i] = static_cast(out_data[i]); - return result; -} - -std::vector WeSpeakerModel::embed(const float* fbank_features, size_t num_features, - const float* mask_weights, size_t mask_num_frames) { - if (!initialized_) throw std::runtime_error("WeSpeaker model not initialized"); - - static constexpr size_t NUM_MEL = 80; - const size_t num_frames = num_features / NUM_MEL; - if (num_frames == 0) throw std::runtime_error("WeSpeaker: empty fbank input"); - - if (num_frames != current_num_frames_ || !current_masked_) { - graph_.hard_reset(); - load_weights_to_graph(&graph_); - build_graph_masked(num_frames); - current_num_frames_ = num_frames; - current_masked_ = true; - } - - input_buf_.resize(num_features); - for (size_t i = 0; i < num_features; ++i) - input_buf_[i] = static_cast<__fp16>(fbank_features[i]); - graph_.set_input(audio_input_, input_buf_.data(), Precision::FP16); - - const auto& mask_shape = graph_.get_output_buffer(mask_input_); - size_t T_post = mask_shape.shape.back(); - - mask_buf_.resize(T_post); - for (size_t t = 0; t < T_post; ++t) { - size_t src_t = (t * mask_num_frames) / T_post; - if (src_t >= mask_num_frames) src_t = mask_num_frames - 1; - mask_buf_[t] = mask_weights[src_t]; - } - graph_.set_input(mask_input_, mask_buf_.data(), Precision::FP32); - - graph_.execute(); - - const auto& out_buf = graph_.get_output_buffer(output_node_); - const __fp16* out_data = out_buf.data_as<__fp16>(); - size_t total = out_buf.total_size; - - std::vector result(total); - for (size_t i = 0; i < total; ++i) - result[i] = static_cast(out_data[i]); - return result; -} - -} -} diff --git a/cactus/models/model_whisper.cpp b/cactus/models/model_whisper.cpp deleted file mode 100644 index 70ea31c92..000000000 --- a/cactus/models/model_whisper.cpp +++ /dev/null @@ -1,778 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include "../npu/npu.h" -#include "../kernel/kernel.h" -#include -#include -#include -#include -#include -#include - -namespace { - -bool has_whisper_cpu_encoder_weights(const std::string& model_folder_path) { - const std::filesystem::path model_path(model_folder_path); - return std::filesystem::exists(model_path / "encoder_conv1_weight.weights") && - std::filesystem::exists(model_path / "encoder_position_embeddings.weights"); -} - -} // namespace - -namespace cactus { -namespace engine { - -WhisperModel::WhisperModel() : Model() {} - -WhisperModel::WhisperModel(const Config& config) : Model(config) { - enc_layers_ = config.num_encoder_layers > 0 ? config.num_encoder_layers : config.num_layers; - dec_layers_ = config.num_decoder_layers > 0 ? config.num_decoder_layers : config.num_layers; - - weight_nodes_.layers.resize(std::max(enc_layers_, dec_layers_)); - - float hd = static_cast(config.attention_head_dim); - if (hd <= 0.0f) { - hd = 64.0f; - } - - attention_scale_ = 1.0f / std::sqrt(hd); - - encoder_block_out_nodes_.resize(enc_layers_, 0); - encoder_k_persistent_.assign(enc_layers_, 0); - encoder_v_persistent_.assign(enc_layers_, 0); - - const float neg_inf = -std::numeric_limits::infinity(); - for (size_t tok : suppress_tokens_) { - suppress_bias_[static_cast(tok)] = neg_inf; - } - suppress_bias_first_step_ = suppress_bias_; - for (size_t tok : begin_suppress_tokens_) { - suppress_bias_first_step_[static_cast(tok)] = neg_inf; - } -} - -void WhisperModel::load_weights_to_graph(CactusGraph* gb) { - - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - - weight_nodes_.decoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/decoder_norm.weights"); - weight_nodes_.decoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/decoder_norm.bias"); - weight_nodes_.decoder_position_embeddings_weight = gb->mmap_weights(model_folder_path_ + "/decoder_position_embeddings.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - if (npu::is_npu_available()) { - std::string npu_encoder_path = model_folder_path_ + "/model.mlpackage"; - npu_encoder_ = npu::create_encoder(); - if (npu_encoder_ && npu_encoder_->load(npu_encoder_path)) { - use_npu_encoder_ = true; - - std::vector typical_input_shape = {1, static_cast(config_.num_mel_bins), 3000}; - npu_encoder_->preallocate(typical_input_shape, "x", ""); - } else { - use_npu_encoder_ = false; - npu_encoder_.reset(); - } - } - - const bool has_cpu_encoder_weights = has_whisper_cpu_encoder_weights(model_folder_path_); - if (has_cpu_encoder_weights) { - weight_nodes_.encoder_position_embeddings = gb->mmap_weights(model_folder_path_ + "/encoder_position_embeddings.weights"); - weight_nodes_.encoder_conv1_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_weight.weights"); - weight_nodes_.encoder_conv1_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv1_bias.bias"); - weight_nodes_.encoder_conv2_weight = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_weight.weights"); - weight_nodes_.encoder_conv2_bias = gb->mmap_weights(model_folder_path_ + "/encoder_conv2_bias.bias"); - weight_nodes_.encoder_norm_weight = gb->mmap_weights(model_folder_path_ + "/encoder_norm_weight.weights"); - weight_nodes_.encoder_norm_bias = gb->mmap_weights(model_folder_path_ + "/encoder_norm_bias.bias"); - } - - const uint32_t max_layers = std::max(enc_layers_, dec_layers_); - for (uint32_t i = 0; i < max_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - - if (i < dec_layers_) { - // Decoder Layers - std::string layer_prefix = model_folder_path_ + "/decoder.layer_" + std::to_string(i) + "_"; - - layer.decoder_encoder_attn_k_weight = gb->mmap_weights(layer_prefix + "encoder_attn_k.weights"); - layer.decoder_encoder_attn_q_weight = gb->mmap_weights(layer_prefix + "encoder_attn_q.weights"); - layer.decoder_encoder_attn_v_weight = gb->mmap_weights(layer_prefix + "encoder_attn_v.weights"); - layer.decoder_encoder_attn_output_weight = gb->mmap_weights(layer_prefix + "encoder_attn_output.weights"); - layer.decoder_encoder_attn_q_bias = gb->mmap_weights(layer_prefix + "encoder_attn_q.bias"); - layer.decoder_encoder_attn_v_bias = gb->mmap_weights(layer_prefix + "encoder_attn_v.bias"); - layer.decoder_encoder_attn_output_bias = gb->mmap_weights(layer_prefix + "encoder_attn_output.bias"); - - layer.decoder_post_encoder_layernorm_weight = gb->mmap_weights(layer_prefix + "encoder_attn_norm.weights"); - layer.decoder_post_encoder_layernorm_bias = gb->mmap_weights(layer_prefix + "encoder_attn_norm.bias"); - - layer.decoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.decoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.decoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.decoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - layer.decoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.decoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); - - layer.decoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.decoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.decoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.decoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.decoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.decoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.decoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - - layer.decoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); - layer.decoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); - } - - if (has_cpu_encoder_weights && i < enc_layers_) { - std::string layer_prefix = model_folder_path_ + "/encoder.layer_" + std::to_string(i) + "_"; - - layer.encoder_ffn1_weight = gb->mmap_weights(layer_prefix + "mlp_fc1.weights"); - layer.encoder_ffn1_bias = gb->mmap_weights(layer_prefix + "mlp_fc1.bias"); - layer.encoder_ffn2_weight = gb->mmap_weights(layer_prefix + "mlp_fc2.weights"); - layer.encoder_ffn2_bias = gb->mmap_weights(layer_prefix + "mlp_fc2.bias"); - - layer.encoder_post_ffn_layernorm_weight = gb->mmap_weights(layer_prefix + "final_norm.weights"); - layer.encoder_post_ffn_layernorm_bias = gb->mmap_weights(layer_prefix + "final_norm.bias"); - - layer.encoder_self_attn_k_weight = gb->mmap_weights(layer_prefix + "self_attn_k.weights"); - layer.encoder_self_attn_q_weight = gb->mmap_weights(layer_prefix + "self_attn_q.weights"); - layer.encoder_self_attn_v_weight = gb->mmap_weights(layer_prefix + "self_attn_v.weights"); - layer.encoder_self_attn_output_weight = gb->mmap_weights(layer_prefix + "self_attn_output.weights"); - layer.encoder_self_attn_q_bias = gb->mmap_weights(layer_prefix + "self_attn_q.bias"); - layer.encoder_self_attn_v_bias = gb->mmap_weights(layer_prefix + "self_attn_v.bias"); - layer.encoder_self_attn_output_bias = gb->mmap_weights(layer_prefix + "self_attn_output.bias"); - - layer.encoder_post_attn_layernorm_weight = gb->mmap_weights(layer_prefix + "self_attn_norm.weights"); - layer.encoder_post_attn_layernorm_bias = gb->mmap_weights(layer_prefix + "self_attn_norm.bias"); - } - } -} - -size_t WhisperModel::build_encoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto ffn1_weight = gb->matmul(input, layer.encoder_ffn1_weight, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, layer.encoder_ffn1_bias); - - encoder_pre_gelu = ffn1_bias; - - auto ffn1_act = gb->gelu_erf(ffn1_bias); - - encoder_post_gelu = ffn1_act; - - auto ffn2_weight = gb->matmul(ffn1_act, layer.encoder_ffn2_weight, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, layer.encoder_ffn2_bias); - return ffn2_bias; -} - -size_t WhisperModel::build_decoder_mlp(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto ffn1_weight = gb->matmul(input, layer.decoder_ffn1_weight, true, backend); - auto ffn1_bias = gb->add(ffn1_weight, layer.decoder_ffn1_bias); - auto ffn1_act = gb->gelu_erf(ffn1_bias); - auto ffn2_weight = gb->matmul(ffn1_act, layer.decoder_ffn2_weight, true, backend); - auto ffn2_bias = gb->add(ffn2_weight, layer.decoder_ffn2_bias); - return ffn2_bias; -} - - - -size_t WhisperModel::build_encoder_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool /*use_cache*/, size_t /*position_offset*/){ - - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t q = gb->matmul(input, layer.decoder_encoder_attn_q_weight, true, backend); - q = gb->add(q, layer.decoder_encoder_attn_q_bias); - - const auto& q_buf = gb->get_output_buffer(q); - if (q_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: q must be [T_dec, D]"); - } - - size_t T_dec = q_buf.shape[0]; - size_t q_heads = config_.attention_heads; - size_t kv_heads = config_.attention_kv_heads; - size_t head_dim = config_.attention_head_dim; - - q = gb->reshape(q, {1, T_dec, q_heads, head_dim}); - - size_t k_4d = 0; - size_t v_4d = 0; - - bool is_populated = (encoder_k_persistent_[layer_idx] != 0 && - gb->is_populated(encoder_k_persistent_[layer_idx])); - - if (is_populated) { - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } else { - size_t enc_norm = last_encoder_post_norm_node_; - - size_t k = gb->matmul(enc_norm, layer.decoder_encoder_attn_k_weight, true, backend); - size_t v = gb->matmul(enc_norm, layer.decoder_encoder_attn_v_weight, true, backend); - v = gb->add(v, layer.decoder_encoder_attn_v_bias); - - const auto& k_buf = gb->get_output_buffer(k); - if (k_buf.shape.size() != 2) { - throw std::runtime_error("encoder cross-attn: k must be [T_enc, D]"); - } - size_t T_enc = k_buf.shape[0]; - - k_4d = gb->reshape(k, {1, T_enc, kv_heads, head_dim}); - v_4d = gb->reshape(v, {1, T_enc, kv_heads, head_dim}); - - if (encoder_k_persistent_[layer_idx] == 0) { - encoder_k_persistent_[layer_idx] = gb->persistent(k_4d); - encoder_v_persistent_[layer_idx] = gb->persistent(v_4d); - } - k_4d = encoder_k_persistent_[layer_idx]; - v_4d = encoder_v_persistent_[layer_idx]; - } - - size_t attn = gb->attention(q, k_4d, v_4d, attention_scale_, false); - - attn = gb->reshape(attn, {T_dec, q_heads * head_dim}); - size_t out = gb->matmul(attn, layer.decoder_encoder_attn_output_weight, true, backend); - out = gb->add(out, layer.decoder_encoder_attn_output_bias); - - return out; -} - -void WhisperModel::reset_graph_side_cache_nodes() { - cache_k_output_nodes_.assign(dec_layers_, 0); - cache_v_output_nodes_.assign(dec_layers_, 0); -} - -void WhisperModel::reset_cache() { - Model::reset_cache(); - encoder_ready_ = false; - first_decode_step_ = true; - - auto* gb = static_cast(graph_handle_); - if (gb) { - if (encoder_output_persistent_ != 0) { - gb->invalidate_persistent(encoder_output_persistent_); - encoder_output_persistent_ = 0; - } - for (size_t i = 0; i < encoder_k_persistent_.size(); ++i) { - if (encoder_k_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_k_persistent_[i]); - encoder_k_persistent_[i] = 0; - } - if (encoder_v_persistent_[i] != 0) { - gb->invalidate_persistent(encoder_v_persistent_[i]); - encoder_v_persistent_[i] = 0; - } - } - } -} - -size_t WhisperModel::build_decoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - auto q = gb->matmul(input, layer.decoder_self_attn_q_weight, true, backend); - q = gb->add(q, layer.decoder_self_attn_q_bias); - - auto k = gb->matmul(input, layer.decoder_self_attn_k_weight, true, backend); - auto v = gb->matmul(input, layer.decoder_self_attn_v_weight, true, backend); - v = gb->add(v, layer.decoder_self_attn_v_bias); - - const auto& q_shape = gb->get_output_buffer(q).shape; - if (q_shape.size() != 2) { - throw std::runtime_error("decoder self-attn: q must be [T_new, D]"); - } - - size_t seq_new = q_shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - size_t num_kv_heads = config_.attention_kv_heads; - - auto q_4d = gb->reshape(q, {1, seq_new, num_heads, head_dim}); - auto k_4d = gb->reshape(k, {1, seq_new, num_kv_heads, head_dim}); - auto v_4d = gb->reshape(v, {1, seq_new, num_kv_heads, head_dim}); - - size_t final_k = k_4d; - size_t final_v = v_4d; - - if (use_cache && !kv_cache_.is_empty()) { - auto k_view = kv_cache_.get_key_view(layer_idx); - auto v_view = kv_cache_.get_value_view(layer_idx); - - if (!k_view.ptr1 || !v_view.ptr1) { - throw std::runtime_error("KV cache view is empty but kv_cache_.is_empty()==false"); - } - - size_t cache_len = kv_cache_.current_seq_len; - - size_t cache_k_node = gb->input( - {1, cache_len, num_kv_heads, head_dim}, - kv_cache_.precision - ); - size_t cache_v_node = gb->input( - {1, cache_len, num_kv_heads, head_dim}, - kv_cache_.precision - ); - - if (k_view.ptr2 == nullptr && v_view.ptr2 == nullptr) { - gb->set_external_input(cache_k_node, const_cast(k_view.ptr1), kv_cache_.precision); - gb->set_external_input(cache_v_node, const_cast(v_view.ptr1), kv_cache_.precision); - } else { - gb->set_external_input(cache_k_node, kv_cache_.get_key_ptr(layer_idx), kv_cache_.precision); - gb->set_external_input(cache_v_node, kv_cache_.get_value_ptr(layer_idx), kv_cache_.precision); - } - - final_k = gb->concat(cache_k_node, k_4d, 1); - final_v = gb->concat(cache_v_node, v_4d, 1); - } - - if (use_cache) { - cache_k_output_nodes_[layer_idx] = final_k; - cache_v_output_nodes_[layer_idx] = final_v; - } else { - cache_k_output_nodes_[layer_idx] = k_4d; - cache_v_output_nodes_[layer_idx] = v_4d; - } - - auto attn_out_4d = gb->attention(q_4d, final_k, final_v, attention_scale_, position_offset); - auto attn_out = gb->reshape(attn_out_4d, {seq_new, num_heads * head_dim}); - - auto output = gb->matmul(attn_out, layer.decoder_self_attn_output_weight, true, backend); - output = gb->add(output, layer.decoder_self_attn_output_bias); - return output; -} - -size_t WhisperModel::build_encoder_self_attention(CactusGraph* gb, size_t input, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t /*position_offset*/){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - if(use_cache) - throw std::runtime_error("The encoder attention layers are not auto-regressive, and thus don't use KV caching!"); - - auto q = gb->matmul(input, layer.encoder_self_attn_q_weight, true, backend); - q = gb->add(q, layer.encoder_self_attn_q_bias); - auto v = gb->matmul(input, layer.encoder_self_attn_v_weight, true, backend); - v = gb->add(v, layer.encoder_self_attn_v_bias); - auto k = gb->matmul(input, layer.encoder_self_attn_k_weight, true, backend); - - size_t seq_len = gb->get_output_buffer(q).shape[0]; - size_t num_heads = config_.attention_heads; - size_t head_dim = config_.attention_head_dim; - - q = gb->reshape(q, {1, seq_len, num_heads, head_dim}); - k = gb->reshape(k, {1, seq_len, num_heads, head_dim}); - v = gb->reshape(v, {1, seq_len, num_heads, head_dim}); - - auto attn = gb->attention(q, k, v, attention_scale_, false); - - attn = gb->reshape(attn, {seq_len, num_heads * head_dim}); - - auto output = gb->matmul(attn, layer.encoder_self_attn_output_weight, true, backend); - output = gb->add(output, layer.encoder_self_attn_output_bias); - - return output; -} - -size_t WhisperModel::build_conv1d(CactusGraph* gb, size_t input) -{ - size_t conv_input = input; - const auto& xbuf = gb->get_output_buffer(input); - - if (xbuf.precision == Precision::INT8) { - conv_input = gb->precision_cast(input, Precision::FP16); - } - - size_t conv1 = gb->conv1d_k3(conv_input, weight_nodes_.encoder_conv1_weight, 1); - - auto bias1_shape = gb->get_output_buffer(weight_nodes_.encoder_conv1_bias).shape; - size_t C1 = bias1_shape[0]; - size_t bias1 = gb->reshape(weight_nodes_.encoder_conv1_bias, {1, C1, 1}); - conv1 = gb->add(conv1, bias1); - - last_conv1_node_ = conv1; - - conv1 = gb->gelu_erf(conv1); - - size_t conv2 = gb->conv1d_k3(conv1, weight_nodes_.encoder_conv2_weight, 2); - auto bias2_shape = gb->get_output_buffer(weight_nodes_.encoder_conv2_bias).shape; - size_t C2 = bias2_shape[0]; - size_t bias2 = gb->reshape(weight_nodes_.encoder_conv2_bias, {1, C2, 1}); - conv2 = gb->add(conv2, bias2); - - last_conv2_node_ = conv2; - - conv2 = gb->gelu_erf(conv2); - - const auto& buf = gb->get_output_buffer(conv2); - - size_t conv2_transposed; - if (buf.precision == Precision::FP16) { - conv2_transposed = gb->transpose(conv2, ComputeBackend::CPU); - } else { - size_t conv2_f16 = gb->precision_cast(conv2, Precision::FP16); - conv2_transposed = gb->transpose(conv2_f16, ComputeBackend::CPU); - } - - return conv2_transposed; -} - -size_t WhisperModel::build_encoder_transformer_block( - CactusGraph* gb, - size_t hidden, - uint32_t layer_idx, - ComputeBackend backend, - bool use_cache, - size_t position_offset) -{ - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ln1 = gb->layernorm( - hidden, - layer.encoder_post_attn_layernorm_weight, - layer.encoder_post_attn_layernorm_bias - ); - - size_t sa = build_encoder_self_attention( - gb, ln1, layer_idx, backend, use_cache, position_offset - ); - - size_t x_post_sa = gb->add(hidden, sa); - - size_t ln2 = gb->layernorm( - x_post_sa, - layer.encoder_post_ffn_layernorm_weight, - layer.encoder_post_ffn_layernorm_bias - ); - - size_t ffn_out = build_encoder_mlp( - gb, ln2, layer_idx, backend - ); - - size_t out = gb->add(x_post_sa, ffn_out); - - if (layer_idx < encoder_block_out_nodes_.size()) { - encoder_block_out_nodes_[layer_idx] = out; - } - - return out; -} - -size_t WhisperModel::build_decoder_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, ComputeBackend backend, bool use_cache, size_t position_offset){ - const auto& layer = weight_nodes_.layers[layer_idx]; - - size_t ln1 = gb->layernorm(hidden, layer.decoder_post_attn_layernorm_weight, layer.decoder_post_attn_layernorm_bias); - size_t sa = build_decoder_self_attention(gb, ln1, layer_idx, backend, use_cache, position_offset); - size_t x_post_sa = gb->add(hidden, sa); - - size_t ln2 = gb->layernorm(x_post_sa, layer.decoder_post_encoder_layernorm_weight, layer.decoder_post_encoder_layernorm_bias); - size_t ca = build_encoder_attention(gb, ln2, layer_idx, backend, use_cache, position_offset); - size_t x_post_ca = gb->add(x_post_sa, ca); - - size_t ln3 = gb->layernorm(x_post_ca,layer.decoder_post_ffn_layernorm_weight,layer.decoder_post_ffn_layernorm_bias); - size_t ffn_out = build_decoder_mlp(gb, ln3, layer_idx, backend); - size_t x_post_ffn = gb->add(x_post_ca, ffn_out); - - return x_post_ffn; - -} - -void WhisperModel::run_encoder(const std::vector& audio_features) -{ - const size_t mel_bins = config_.num_mel_bins; - if (audio_features.size() % mel_bins != 0) - throw std::runtime_error("Mel bins length must be divisible by " + std::to_string(mel_bins) + "."); - - size_t T_mel = audio_features.size() / mel_bins; - if (T_mel == 0) - throw std::runtime_error("Mel bins has zero frames."); - - auto* gb = static_cast(graph_handle_); - if (!gb) - throw std::runtime_error("Graph handle is null in run_encoder."); - - if (use_npu_encoder_ && npu_encoder_ && npu_encoder_->is_available()) { - std::vector out_shape = npu_encoder_->get_output_shape(); - size_t T_enc, D_enc; - if (out_shape.size() == 3) { - T_enc = static_cast(out_shape[1]); - D_enc = static_cast(out_shape[2]); - } else if (out_shape.size() == 2) { - T_enc = static_cast(out_shape[0]); - D_enc = static_cast(out_shape[1]); - } else { - throw std::runtime_error("NPU encoder output has unexpected shape"); - } - - std::vector<__fp16> audio_features_f16(audio_features.size()); - cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); - - std::vector input_shape = {1, static_cast(mel_bins), static_cast(T_mel)}; - size_t total_elements = T_enc * D_enc; - std::vector<__fp16> npu_output(total_elements); - size_t elements_written = npu_encoder_->encode( - audio_features_f16.data(), - npu_output.data(), - input_shape, - "x", - "" - ); - - if (elements_written > 0) { - size_t enc_output_node = gb->input({T_enc, D_enc}, Precision::FP16); - gb->set_input(enc_output_node, npu_output.data(), Precision::FP16); - - size_t enc_output_persistent = gb->persistent(enc_output_node); - encoder_output_persistent_ = enc_output_persistent; - last_encoder_post_norm_node_ = enc_output_persistent; - return; - } - } - - if (!has_whisper_cpu_encoder_weights(model_folder_path_)) { - throw std::runtime_error( - "Whisper requires either CPU encoder weights or model.mlpackage encoder output."); - } - - auto backend = - (config_.default_backend == Config::Backend::CPU) - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t mel_input = 0; - std::vector<__fp16> audio_features_f16(audio_features.size()); - cactus_fp32_to_fp16(audio_features.data(), audio_features_f16.data(), audio_features.size()); - - mel_input = gb->input({1, mel_bins, T_mel}, Precision::FP16); - gb->set_input(mel_input, audio_features_f16.data(), Precision::FP16); - - size_t conv2_transposed = build_conv1d(gb, mel_input); - - const auto& conv_shape = gb->get_output_buffer(conv2_transposed).shape; - if (conv_shape.size() != 3 || conv_shape[0] != 1) - throw std::runtime_error("Conv2 transpose should be [1, T_enc, D]."); - - size_t T_enc = conv_shape[1]; - size_t D_enc = conv_shape[2]; - - size_t pos_slice = gb->slice(weight_nodes_.encoder_position_embeddings, 0, 0, T_enc); - - size_t h2d = gb->reshape(conv2_transposed, {T_enc, D_enc}); - - auto& h2d_buf = gb->get_output_buffer(h2d); - auto& pos_buf = gb->get_output_buffer(pos_slice); - - if (pos_buf.precision != h2d_buf.precision) { - pos_slice = gb->precision_cast(pos_slice, h2d_buf.precision); - } - - size_t h_pos = gb->add(h2d, pos_slice); - last_enc_plus_pos_node_ = h_pos; - - size_t h = h_pos; - for (uint32_t i = 0; i < enc_layers_; ++i){ - h = build_encoder_transformer_block(gb, h, i, backend, false, 0); - if (i == 0) { - encoder_transformer_block_0 = h; - } - } - - size_t h_norm = gb->layernorm( - h, - weight_nodes_.encoder_norm_weight, - weight_nodes_.encoder_norm_bias - ); - - size_t h_norm_persistent = gb->persistent(h_norm); - encoder_output_persistent_ = h_norm_persistent; - last_encoder_post_norm_node_ = h_norm_persistent; -} - - - -size_t WhisperModel::run_decoder_step(const std::vector& tokens, bool use_cache, bool last_token_only) { - auto* gb = static_cast(graph_handle_); - - const size_t full_len = tokens.size(); - if (full_len == 0) { - throw std::runtime_error("Decoder token list cannot be empty."); - } - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - size_t start_idx = (use_cache && kv_cache_.current_seq_len > 0) ? full_len - 1 : 0; - size_t new_tokens = full_len - start_idx; - - size_t tok_input = gb->input({new_tokens}, Precision::FP32); - std::vector tok_f(new_tokens); - for (size_t i = 0; i < new_tokens; i++) { - tok_f[i] = static_cast(tokens[start_idx + i]); - } - gb->set_input(tok_input, tok_f.data(), Precision::FP32); - - size_t dec_hidden = gb->embedding(embedding_node_id_, tok_input); - - size_t position_offset = kv_cache_.current_seq_len; - size_t dec_pos = gb->slice(weight_nodes_.decoder_position_embeddings_weight,0,position_offset,new_tokens); - - { - const auto& h_buf = gb->get_output_buffer(dec_hidden); - const auto& pos_buf = gb->get_output_buffer(dec_pos); - - size_t pos_node_for_add = dec_pos; - if (pos_buf.precision != h_buf.precision) { - pos_node_for_add = gb->precision_cast(dec_pos, h_buf.precision); - } - - dec_hidden = gb->add(dec_hidden, pos_node_for_add); - } - - for (uint32_t layer_idx = 0; layer_idx < dec_layers_; ++layer_idx) { - dec_hidden = build_decoder_transformer_block( - gb, - dec_hidden, - layer_idx, - backend, - use_cache, - position_offset - ); - } - - size_t dec_norm = gb->layernorm( - dec_hidden, - weight_nodes_.decoder_norm_weight, - weight_nodes_.decoder_norm_bias - ); - - size_t logits_input = dec_norm; - if (last_token_only) { - size_t row_index = new_tokens - 1; - logits_input = gb->slice(logits_input, 0, row_index, 1); - } - - auto w_shape = gb->get_output_buffer(output_weight_node_id_).shape; - - size_t logits = gb->matmul(logits_input, output_weight_node_id_, true, backend); - - last_new_tokens_ = new_tokens; - return logits; -} - - -size_t WhisperModel::forward(const std::vector& audio_features, const std::vector& tokens, bool use_cache) -{ - - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized"); - } - - if (!use_cache) { - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - run_encoder(audio_features); - } - - return run_decoder_step(tokens, use_cache, false); -} - -std::vector WhisperModel::get_audio_embeddings(const std::vector& audio_features) { - run_encoder(audio_features); - - auto* gb = static_cast(graph_handle_); - - size_t pooled = gb->mean(encoder_output_persistent_, 0); - gb->execute(); - - const auto& output_buf = gb->get_output_buffer(pooled); - size_t hidden_dim = output_buf.total_size; - - std::vector embedding(hidden_dim); - void* output_data = gb->get_output(pooled); - const float* output_ptr = static_cast(output_data); - std::copy(output_ptr, output_ptr + hidden_dim, embedding.begin()); - - reset_cache(); - return embedding; -} - -uint32_t WhisperModel::decode_with_audio( - const std::vector& tokens, - const std::vector& audio_features, - float temperature, - float top_p, - size_t top_k, - const std::string& profile_file, - float* out_entropy, - float min_p, - float repetition_penalty, - float* /*out_token_time_start*/, - float* /*out_token_time_end*/) -{ - if (!initialized_ || !graph_handle_) - throw std::runtime_error("Model not initialized - call init() first"); - if (audio_features.empty()) - throw std::runtime_error("Mel bins cannot be empty in Whisper decode_with_audio"); - - auto* gb = static_cast(graph_handle_); - - bool cold_start = !encoder_ready_; - size_t logits_node = 0; - - std::vector full_tokens; - if (tokens.empty()) { - full_tokens.push_back(static_cast(get_tokenizer()->get_bos_token())); - } else { - full_tokens = tokens; - } - - if (cold_start) - { - gb->soft_reset(); - kv_cache_.reset(); - kv_cache_.current_seq_len = 0; - reset_graph_side_cache_nodes(); - - first_decode_step_ = true; - run_encoder(audio_features); - logits_node = run_decoder_step(full_tokens, false, false); - encoder_ready_ = true; - } - else - { - gb->soft_reset(); - reset_graph_side_cache_nodes(); - - std::vector last_token_vec = { tokens.back() }; - logits_node = run_decoder_step(last_token_vec, true, true); - } - - const auto& suppress_bias = first_decode_step_ ? suppress_bias_first_step_ : suppress_bias_; - if (first_decode_step_) first_decode_step_ = false; - size_t sampled_token_id = sample_token(gb, logits_node, temperature, top_p, top_k, min_p, repetition_penalty, &suppress_bias); - if (!profile_file.empty()) gb->execute(profile_file); - else gb->execute(); - - - compute_entropy(gb, logits_node, out_entropy); - - post_execute_updates(gb, full_tokens.size()); - update_kv_cache(gb, last_new_tokens_); - - auto* out_ptr = gb->get_output(sampled_token_id); - uint32_t sampled = *reinterpret_cast(out_ptr); - record_sampled_token(sampled); - - return sampled; -} - - -} -} diff --git a/cactus/models/model_youtu.cpp b/cactus/models/model_youtu.cpp deleted file mode 100644 index cfd1028ab..000000000 --- a/cactus/models/model_youtu.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include "model.h" -#include "../graph/graph.h" -#include -#include - -namespace cactus { -namespace engine { - -YoutuModel::YoutuModel() : Model() {} - -YoutuModel::YoutuModel(const Config& config) : Model(config) { - weight_nodes_.layers.resize(config.num_layers); -} - -std::vector YoutuModel::get_kv_layer_dims() const { - return std::vector(config_.num_layers, static_cast(config_.qk_head_dim)); -} - -void YoutuModel::post_init() { - std::vector v_dims(config_.num_layers, static_cast(config_.v_head_dim)); - std::vector v_kv_heads(config_.num_layers, static_cast(config_.attention_kv_heads)); - v_cache_.init(config_.num_layers, kv_cache_.max_seq_len, - v_dims, v_kv_heads, Precision::INT8); - v_cache_.set_window_size(kv_cache_.window_size, kv_cache_.sink_size); - cache_v_nodes_.resize(config_.num_layers, 0); -} - -void YoutuModel::post_execute_updates(CactusGraph* gb, size_t seq_len) { - v_cache_.update_from_graph(gb, cache_v_nodes_, cache_v_nodes_, seq_len, - config_.num_layers); -} - -void YoutuModel::reset_cache() { - Model::reset_cache(); - v_cache_.reset(); -} - -void YoutuModel::load_weights_to_graph(CactusGraph* gb) { - embedding_node_id_ = gb->mmap_embeddings(embedding_file_path_); - weight_nodes_.output_norm_weight = gb->mmap_weights(model_folder_path_ + "/output_norm.weights"); - - if (config_.tie_word_embeddings) { - weight_nodes_.output_weight = embedding_node_id_; - output_weight_node_id_ = embedding_node_id_; - } else { - weight_nodes_.output_weight = gb->mmap_weights(model_folder_path_ + "/output_weight.weights"); - output_weight_node_id_ = weight_nodes_.output_weight; - } - - for (uint32_t i = 0; i < config_.num_layers; i++) { - auto& layer = weight_nodes_.layers[i]; - std::string layer_prefix = model_folder_path_ + "/layer_" + std::to_string(i) + "_"; - if (config_.q_lora_rank == 0) { - layer.attn_q_weight = gb->mmap_weights(layer_prefix + "attn_q.weights"); - } else { - layer.attn_q_a_weight = gb->mmap_weights(layer_prefix + "attn_q_a.weights"); - layer.attn_q_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_q_a_norm.weights"); - layer.attn_q_b_weight = gb->mmap_weights(layer_prefix + "attn_q_b.weights"); - } - layer.attn_kv_a_weight = gb->mmap_weights(layer_prefix + "attn_kv_a.weights"); - layer.attn_kv_a_norm_weight = gb->mmap_weights(layer_prefix + "attn_kv_a_norm.weights"); - layer.attn_kv_b_weight = gb->mmap_weights(layer_prefix + "attn_kv_b.weights"); - layer.attn_output_weight = gb->mmap_weights(layer_prefix + "attn_output.weights"); - if (config_.attention_bias) { - layer.attn_q_a_bias = gb->mmap_weights(layer_prefix + "attn_q_a_bias.weights"); - layer.attn_kv_a_bias = gb->mmap_weights(layer_prefix + "attn_kv_a_bias.weights"); - layer.attn_output_bias = gb->mmap_weights(layer_prefix + "attn_output_bias.weights"); - } - layer.input_layernorm_weight = gb->mmap_weights(layer_prefix + "input_norm.weights"); - layer.post_attention_layernorm_weight = gb->mmap_weights(layer_prefix + "post_attn_norm.weights"); - layer.ffn_gate_weight = gb->mmap_weights(layer_prefix + "ffn_gate.weights"); - layer.ffn_up_weight = gb->mmap_weights(layer_prefix + "ffn_up.weights"); - layer.ffn_down_weight = gb->mmap_weights(layer_prefix + "ffn_down.weights"); - } -} - -static float yarn_get_mscale(float scale, float mscale) { - if (scale <= 1.0f) return 1.0f; - return 0.1f * mscale * logf(scale) + 1.0f; -} - -size_t YoutuModel::build_attention(CactusGraph* gb, size_t normalized_input, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - - const size_t seq_len = gb->get_output_buffer(normalized_input).shape[0]; - const size_t num_heads = config_.attention_heads; - const size_t num_kv_heads = config_.attention_kv_heads; - const size_t kv_lora = config_.kv_lora_rank; - const size_t qk_head = config_.qk_head_dim; - const size_t qk_nope = config_.qk_nope_head_dim; - const size_t qk_rope = config_.qk_rope_head_dim; - const size_t v_dim = config_.v_head_dim; - const float eps = config_.layer_norm_eps; - - size_t q_full; - if (config_.q_lora_rank == 0) { - q_full = gb->matmul(normalized_input, layer.attn_q_weight, true, backend); - } else { - auto q_latent = gb->matmul(normalized_input, layer.attn_q_a_weight, true, backend); - if (config_.attention_bias) { - q_latent = gb->add(q_latent, layer.attn_q_a_bias); - } - q_latent = gb->rms_norm(q_latent, layer.attn_q_a_norm_weight, eps); - q_full = gb->matmul(q_latent, layer.attn_q_b_weight, true, backend); - } - - q_full = gb->reshape(q_full, {seq_len * num_heads, qk_head}); - auto q_nope = gb->slice(q_full, 1, 0, qk_nope); - auto q_rope_raw = gb->slice(q_full, 1, qk_nope, qk_rope); - - q_rope_raw = gb->reshape(q_rope_raw, {1, seq_len, num_heads, qk_rope}); - size_t q_rope_rotated; - if (config_.rope_interleave) { - q_rope_rotated = gb->rope_gptj(q_rope_raw, config_.rope_theta, position_offset, qk_rope); - } else { - q_rope_rotated = gb->rope(q_rope_raw, config_.rope_theta, position_offset); - } - q_rope_rotated = gb->reshape(q_rope_rotated, {seq_len * num_heads, qk_rope}); - - auto q_combined = gb->concat(q_nope, q_rope_rotated, -1); - auto q_4d = gb->reshape(q_combined, {1, seq_len, num_heads, qk_head}); - - auto kv_combined = gb->matmul(normalized_input, layer.attn_kv_a_weight, true, backend); - if (config_.attention_bias) { - kv_combined = gb->add(kv_combined, layer.attn_kv_a_bias); - } - auto kv_latent = gb->slice(kv_combined, 1, 0, kv_lora); - auto k_rope_raw = gb->slice(kv_combined, 1, kv_lora, qk_rope); - - kv_latent = gb->rms_norm(kv_latent, layer.attn_kv_a_norm_weight, eps); - - auto kv_decoded = gb->matmul(kv_latent, layer.attn_kv_b_weight, true, backend); - kv_decoded = gb->reshape(kv_decoded, {seq_len * num_kv_heads, qk_nope + v_dim}); - auto k_nope = gb->slice(kv_decoded, 1, 0, qk_nope); - auto v_flat = gb->slice(kv_decoded, 1, qk_nope, v_dim); - - k_rope_raw = gb->reshape(k_rope_raw, {1, seq_len, 1, qk_rope}); - size_t k_rope_rotated; - if (config_.rope_interleave) { - k_rope_rotated = gb->rope_gptj(k_rope_raw, config_.rope_theta, position_offset, qk_rope); - } else { - k_rope_rotated = gb->rope(k_rope_raw, config_.rope_theta, position_offset); - } - std::vector k_rope_copies(num_kv_heads, k_rope_rotated); - auto k_rope_4d = gb->cat(k_rope_copies, 2); - auto k_rope_flat = gb->reshape(k_rope_4d, {seq_len * num_kv_heads, qk_rope}); - - auto k_combined = gb->concat(k_nope, k_rope_flat, -1); - auto k_4d = gb->reshape(k_combined, {1, seq_len, num_kv_heads, qk_head}); - auto v_4d = gb->reshape(v_flat, {1, seq_len, num_kv_heads, v_dim}); - - if (use_cache && layer_idx < cache_k_output_nodes_.size()) { - cache_k_output_nodes_[layer_idx] = gb->reshape(k_4d, {seq_len * num_kv_heads, qk_head}); - cache_v_output_nodes_[layer_idx] = cache_k_output_nodes_[layer_idx]; - cache_v_nodes_[layer_idx] = gb->reshape(v_4d, {seq_len * num_kv_heads, v_dim}); - } - - float scale = 1.0f / sqrtf(static_cast(qk_head)); - if (config_.rope_mscale_all_dim != 0.0f && config_.rope_scaling_factor > 1.0f) { - float mscale = yarn_get_mscale(config_.rope_scaling_factor, config_.rope_mscale_all_dim); - scale *= mscale * mscale; - } - - size_t attn_4d; - if (use_cache && !kv_cache_.is_empty()) { - attn_4d = gb->attention_int8_hybrid( - q_4d, k_4d, v_4d, scale, position_offset, - kv_cache_.get_keys_int8(layer_idx), - v_cache_.get_values_int8(layer_idx), - kv_cache_.get_key_scales(layer_idx), - v_cache_.get_value_scales(layer_idx), - kv_cache_.current_seq_len, num_kv_heads, qk_head, - kv_cache_.window_size, v_dim); - } else { - attn_4d = gb->attention(q_4d, k_4d, v_4d, scale, position_offset); - } - - auto attn_out = gb->reshape(attn_4d, {seq_len, num_heads * v_dim}); - size_t out = gb->matmul(attn_out, layer.attn_output_weight, true, backend); - if (config_.attention_bias) { - out = gb->add(out, layer.attn_output_bias); - } - return out; -} - - -size_t YoutuModel::build_mlp(CactusGraph* gb, size_t normalized_h, uint32_t layer_idx, - ComputeBackend backend) const { - const auto& layer = weight_nodes_.layers[layer_idx]; - size_t gate_output = gb->matmul(normalized_h, layer.ffn_gate_weight, true, backend); - size_t up_output = gb->matmul(normalized_h, layer.ffn_up_weight, true, backend); - size_t gate_silu = gb->silu(gate_output); - size_t gated = gb->multiply(gate_silu, up_output); - return gb->matmul(gated, layer.ffn_down_weight, true, backend); -} - - -size_t YoutuModel::build_transformer_block(CactusGraph* gb, size_t hidden, uint32_t layer_idx, - ComputeBackend backend, bool use_cache, size_t position_offset) { - const auto& layer = weight_nodes_.layers[layer_idx]; - auto normalized_input = gb->rms_norm(hidden, layer.input_layernorm_weight, config_.layer_norm_eps); - auto attn_output = build_attention(gb, normalized_input, layer_idx, backend, use_cache, position_offset); - auto after_attention = gb->add_clipped(hidden, attn_output); - auto normalized_after_attention = gb->rms_norm(after_attention, layer.post_attention_layernorm_weight, config_.layer_norm_eps); - auto mlp_output = build_mlp(gb, normalized_after_attention, layer_idx, backend); - return gb->add_clipped(after_attention, mlp_output); -} - - -size_t YoutuModel::forward(const std::vector& tokens, bool use_cache) { - if (!initialized_ || !graph_handle_) { - throw std::runtime_error("Model not initialized - call init() first"); - } - if (tokens.empty()) { - throw std::runtime_error("Token sequence cannot be empty"); - } - - auto* gb = static_cast(graph_handle_); - gb->soft_reset(); - - const size_t seq_len = tokens.size(); - const size_t position_offset = use_cache ? kv_cache_.get_total_seq_len() : 0; - - auto backend = config_.default_backend == Config::Backend::CPU - ? ComputeBackend::CPU - : ComputeBackend::NPU; - - auto input_node_id = gb->input({seq_len}, Precision::FP32); - auto hidden = gb->embedding(embedding_node_id_, input_node_id); - - std::vector input_data(seq_len); - for (size_t i = 0; i < seq_len; i++) { - input_data[i] = static_cast(tokens[i]); - } - gb->set_input(input_node_id, input_data.data(), Precision::FP32); - - for (uint32_t layer_idx = 0; layer_idx < config_.num_layers; layer_idx++) { - hidden = build_transformer_block(gb, hidden, layer_idx, backend, use_cache, position_offset); - } - - return gb->rms_norm(hidden, weight_nodes_.output_norm_weight, config_.layer_norm_eps); -} - -} -} diff --git a/cactus/npu/npu.cpp b/cactus/npu/npu.cpp deleted file mode 100644 index 69a9f80c7..000000000 --- a/cactus/npu/npu.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// NPU stub implementation for non-Apple platforms -// On Apple platforms, npu_ane.mm provides the real implementation - -#include "npu.h" - -namespace cactus { -namespace npu { - -std::unique_ptr create_encoder() { - return nullptr; -} - -std::unique_ptr create_prefill() { - return nullptr; -} - -bool is_npu_available() { - return false; -} - -} // namespace npu -} // namespace cactus diff --git a/cactus/npu/npu.h b/cactus/npu/npu.h deleted file mode 100644 index 5474c872f..000000000 --- a/cactus/npu/npu.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef CACTUS_NPU_H -#define CACTUS_NPU_H - -#include -#include -#include -#include - -namespace cactus { -namespace npu { - - -struct NPUNamedInput { - std::string name; - const __fp16* data; - std::vector shape; -}; - -class NPUEncoder { -public: - virtual ~NPUEncoder() = default; - - virtual bool load(const std::string& model_path) = 0; - - virtual bool preallocate(const std::vector& input_shape, - const std::string& input_name = "x", - const std::string& output_name = "") = 0; - - virtual size_t encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name = "x", - const std::string& output_name = "") = 0; - - virtual bool is_available() const = 0; - - virtual std::vector get_input_shape() const = 0; - - virtual std::vector get_output_shape() const = 0; - - virtual __fp16* get_output_buffer() = 0; - - virtual size_t get_output_buffer_size() const = 0; - - virtual size_t encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name = "") = 0; -}; - -std::unique_ptr create_encoder(); - -bool is_npu_available(); - -struct NPUBufferRef { - const __fp16* data; - size_t count; -}; - -struct NPUPrefillDirectResult { - NPUBufferRef hidden; - std::vector k_caches; - std::vector v_caches; - bool valid; -}; - -class NPUPrefill { -public: - virtual ~NPUPrefill() = default; - virtual bool load(const std::string& model_path) = 0; - virtual bool is_available() const = 0; - virtual int get_chunk_size() const = 0; - virtual int get_hidden_dim() const = 0; - virtual int get_num_layers() const = 0; - virtual int get_num_kv_heads() const = 0; - virtual int get_head_dim() const = 0; - - virtual NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") = 0; -}; - -std::unique_ptr create_prefill(); - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_NPU_H \ No newline at end of file diff --git a/cactus/npu/npu_ane.h b/cactus/npu/npu_ane.h deleted file mode 100644 index c590fb0c4..000000000 --- a/cactus/npu/npu_ane.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef CACTUS_NPU_ANE_H -#define CACTUS_NPU_ANE_H - -#include "npu.h" - -#if defined(__APPLE__) -#define CACTUS_HAS_ANE 1 -#else -#define CACTUS_HAS_ANE 0 -#endif - -namespace cactus { -namespace npu { - -#if CACTUS_HAS_ANE - - -class ANEEncoder : public NPUEncoder { -public: - ANEEncoder(); - ~ANEEncoder() override; - - ANEEncoder(const ANEEncoder&) = delete; - ANEEncoder& operator=(const ANEEncoder&) = delete; - - ANEEncoder(ANEEncoder&& other) noexcept; - ANEEncoder& operator=(ANEEncoder&& other) noexcept; - - bool load(const std::string& model_path) override; - - bool preallocate(const std::vector& input_shape, - const std::string& input_name = "x", - const std::string& output_name = "") override; - - size_t encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name = "x", - const std::string& output_name = "") override; - - bool is_available() const override; - - std::vector get_input_shape() const override; - - std::vector get_output_shape() const override; - - __fp16* get_output_buffer() override; - - size_t get_output_buffer_size() const override; - - size_t encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name = "") override; - -private: - void* impl_; -}; - -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill(); - ~ANEPrefill() override; - - ANEPrefill(const ANEPrefill&) = delete; - ANEPrefill& operator=(const ANEPrefill&) = delete; - - ANEPrefill(ANEPrefill&& other) noexcept; - ANEPrefill& operator=(ANEPrefill&& other) noexcept; - - bool load(const std::string& model_path) override; - bool is_available() const override; - - int get_chunk_size() const override; - int get_hidden_dim() const override; - int get_num_layers() const override; - int get_num_kv_heads() const override; - int get_head_dim() const override; - - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset = 0, - const std::string& input_name = "x") override; - -private: - void* impl_; - int chunk_size_ = 256; - int hidden_dim_ = 0; - int num_layers_ = 0; - int num_kv_heads_ = 0; - int head_dim_ = 0; -}; - -#else - -class ANEEncoder : public NPUEncoder { -public: - ANEEncoder() = default; - ~ANEEncoder() override = default; - - bool load(const std::string&) override { return false; } - - bool preallocate(const std::vector&, - const std::string& = "x", - const std::string& = "") override { return false; } - - size_t encode(const __fp16*, - __fp16*, - const std::vector&, - const std::string& = "x", - const std::string& = "") override { return 0; } - - bool is_available() const override { return false; } - - std::vector get_input_shape() const override { return {}; } - - std::vector get_output_shape() const override { return {}; } - - __fp16* get_output_buffer() override { return nullptr; } - - size_t get_output_buffer_size() const override { return 0; } - - size_t encode_multimodal_input( - const std::vector&, - __fp16*, - const std::string& = "") override { return 0; } -}; - -class ANEPrefill : public NPUPrefill { -public: - ANEPrefill() = default; - ~ANEPrefill() override = default; - - bool load(const std::string&) override { return false; } - bool is_available() const override { return false; } - - int get_chunk_size() const override { return 0; } - int get_hidden_dim() const override { return 0; } - int get_num_layers() const override { return 0; } - int get_num_kv_heads() const override { return 0; } - int get_head_dim() const override { return 0; } - - NPUPrefillDirectResult prefill_chunk_direct( - const std::vector<__fp16>&, - int = 0, - const std::string& = "x") override { return {}; } -}; - -#endif // CACTUS_HAS_ANE - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_NPU_ANE_H \ No newline at end of file diff --git a/cactus/npu/npu_ane.mm b/cactus/npu/npu_ane.mm deleted file mode 100644 index 488d43d6b..000000000 --- a/cactus/npu/npu_ane.mm +++ /dev/null @@ -1,1293 +0,0 @@ -#include "npu_ane.h" - -#if CACTUS_HAS_ANE - -#import -#import -#include -#include -#include -#include -#include -#include -#include -#include "../graph/graph.h" - -namespace { - -static std::string to_lower_ascii(std::string value) { - std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { - return static_cast(std::tolower(c)); - }); - return value; -} - -static bool parse_compute_units(const char* raw_value, MLComputeUnits& out_units) { - if (!raw_value || raw_value[0] == '\0') { - return false; - } - - std::string value = to_lower_ascii(raw_value); - if (value == "all" || value == "default") { - out_units = MLComputeUnitsAll; - return true; - } - if (value == "cpu_and_ne" || value == "cpu-ne" || value == "cpu_ne" || - value == "ne" || value == "ane" || value == "cpuandneuralengine") { - out_units = MLComputeUnitsCPUAndNeuralEngine; - return true; - } - if (value == "cpu_and_gpu" || value == "cpu_gpu" || value == "cpu-gpu" || - value == "cpuandgpu") { - out_units = MLComputeUnitsCPUAndGPU; - return true; - } - if (value == "cpu_only" || value == "cpu-only" || value == "cpu") { - out_units = MLComputeUnitsCPUOnly; - return true; - } - - return false; -} - -static const char* compute_units_to_string(MLComputeUnits units) { - switch (units) { - case MLComputeUnitsAll: - return "ALL"; - case MLComputeUnitsCPUAndGPU: - return "CPU_AND_GPU"; - case MLComputeUnitsCPUOnly: - return "CPU_ONLY"; - case MLComputeUnitsCPUAndNeuralEngine: - return "CPU_AND_NE"; - } - return "UNKNOWN"; -} - -static bool model_path_looks_like_audio_encoder(NSString* model_path) { - if (!model_path) return false; - NSString* lower = [[model_path lastPathComponent] lowercaseString]; - return [lower containsString:@"audio_encoder"]; -} - -static bool model_path_looks_like_vision_encoder(NSString* model_path) { - if (!model_path) return false; - NSString* lower = [[model_path lastPathComponent] lowercaseString]; - return [lower containsString:@"vision_encoder"]; -} - -static void maybe_apply_compute_units_env(const char* env_name, MLComputeUnits& target) { - const char* raw = std::getenv(env_name); - if (!raw || raw[0] == '\0') return; - - MLComputeUnits parsed; - if (parse_compute_units(raw, parsed)) { - target = parsed; - return; - } - - CACTUS_LOG_WARN("npu", "Ignoring invalid " << env_name << "=" << raw); -} - -static MLComputeUnits resolve_compute_units_for_model(NSString* model_path, bool is_prefill) { - MLComputeUnits units = MLComputeUnitsCPUAndNeuralEngine; - - // Gemma4 multimodal encoders show better fidelity on CPU+GPU. - if (!is_prefill && model_path_looks_like_audio_encoder(model_path)) { - units = MLComputeUnitsCPUAndGPU; - } - if (!is_prefill && model_path_looks_like_vision_encoder(model_path)) { - units = MLComputeUnitsCPUAndGPU; - } - - maybe_apply_compute_units_env("CACTUS_ANE_COMPUTE_UNITS", units); - if (is_prefill) { - maybe_apply_compute_units_env("CACTUS_ANE_PREFILL_COMPUTE_UNITS", units); - } else { - maybe_apply_compute_units_env("CACTUS_ANE_ENCODER_COMPUTE_UNITS", units); - if (model_path_looks_like_audio_encoder(model_path)) { - maybe_apply_compute_units_env("CACTUS_ANE_AUDIO_COMPUTE_UNITS", units); - } - if (model_path_looks_like_vision_encoder(model_path)) { - maybe_apply_compute_units_env("CACTUS_ANE_VISION_COMPUTE_UNITS", units); - } - } - - return units; -} - -static bool should_recompile_mlpackage(NSString* mlpackage_path, NSString* mlmodelc_path) { - NSFileManager* fm = [NSFileManager defaultManager]; - if (![fm fileExistsAtPath:mlmodelc_path]) return true; - - const char* force = std::getenv("CACTUS_ANE_FORCE_RECOMPILE"); - if (force && force[0] != '\0' && std::strcmp(force, "0") != 0) { - return true; - } - - NSError* pkg_err = nil; - NSError* cache_err = nil; - NSDictionary* pkg_attr = [fm attributesOfItemAtPath:mlpackage_path error:&pkg_err]; - NSDictionary* cache_attr = [fm attributesOfItemAtPath:mlmodelc_path error:&cache_err]; - if (!pkg_attr || !cache_attr || pkg_err || cache_err) { - return false; - } - - NSDate* pkg_mtime = pkg_attr[NSFileModificationDate]; - NSDate* cache_mtime = cache_attr[NSFileModificationDate]; - if (!pkg_mtime || !cache_mtime) { - return false; - } - return ([pkg_mtime compare:cache_mtime] == NSOrderedDescending); -} - -static NSURL* resolve_or_compile_model_url(NSString* path, NSError** error) { - NSURL* modelURL = [NSURL fileURLWithPath:path]; - if (![path hasSuffix:@".mlpackage"]) { - return modelURL; - } - - BOOL isDir = NO; - if (![[NSFileManager defaultManager] fileExistsAtPath:path isDirectory:&isDir] || !isDir) { - CACTUS_LOG_ERROR("npu", "ANE mlpackage path is not a valid directory: " << [path UTF8String]); - return modelURL; - } - - NSString* cachedPath = [[path stringByDeletingPathExtension] stringByAppendingPathExtension:@"mlmodelc"]; - NSURL* cachedURL = [NSURL fileURLWithPath:cachedPath]; - NSFileManager* fm = [NSFileManager defaultManager]; - - if (!should_recompile_mlpackage(path, cachedPath)) { - return cachedURL; - } - - if ([fm fileExistsAtPath:cachedPath]) { - NSError* rmErr = nil; - [fm removeItemAtPath:cachedPath error:&rmErr]; - if (rmErr) { - CACTUS_LOG_WARN("npu", "Failed to remove stale mlmodelc: " << [[rmErr localizedDescription] UTF8String]); - } else { - CACTUS_LOG_INFO("npu", "Removed stale mlmodelc cache, recompiling: " << [cachedPath UTF8String]); - } - } - - NSURL* compiledURL = [MLModel compileModelAtURL:modelURL error:error]; - if (*error || !compiledURL) { - CACTUS_LOG_ERROR("npu", "ANE model compilation failed: " << [[*error localizedDescription] UTF8String]); - return modelURL; - } - - NSError* moveError = nil; - [fm moveItemAtURL:compiledURL toURL:cachedURL error:&moveError]; - if (moveError) { - CACTUS_LOG_WARN("npu", "Could not persist compiled mlmodelc cache: " << [[moveError localizedDescription] UTF8String]); - return compiledURL; - } - return cachedURL; -} - -} // namespace - -@interface CactusANEImpl : NSObject - -@property (nonatomic, strong) MLModel* model; -@property (nonatomic, strong) MLModelDescription* modelDescription; -@property (nonatomic, strong) MLMultiArray* cachedInputArray; -@property (nonatomic, strong) MLMultiArray* cachedOutputArray; -@property (nonatomic, strong) NSArray* cachedShape; -@property (nonatomic, strong) NSArray* cachedOutputShape; -@property (nonatomic, strong) NSString* cachedInputName; -@property (nonatomic, strong) NSString* cachedOutputName; -@property (nonatomic, assign) NSUInteger cachedOutputSize; -@property (nonatomic, strong) MLPredictionOptions* predictionOptions; -@property (nonatomic, strong) NSMutableDictionary* cachedMultiInputArrays; -@property (nonatomic, strong) MLMultiArray* cachedMultiOutputArray; -@property (nonatomic, strong) MLPredictionOptions* multiPredictionOptions; -@property (nonatomic, assign) BOOL multiInputPreallocated; -@property (nonatomic, assign) BOOL hasOutputBackings; -@property (nonatomic, assign) BOOL hasMultiOutputBackings; - -- (instancetype)initWithModelPath:(NSString*)path; -- (NSArray*)getInputShape; -- (NSArray*)getOutputShape; -- (BOOL)preallocateBuffersWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (MLMultiArray*)predictWithInput:(NSString*)inputName - data:(const __fp16*)data - shape:(NSArray*)shape - outputName:(NSString*)outputName; -- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName; -- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict - outputData:(__fp16*)output - outputName:(NSString*)outputName; - -@end - -@implementation CactusANEImpl - -- (instancetype)initWithModelPath:(NSString*)path { - self = [super init]; - if (self) { - NSError* error = nil; - NSURL* modelURL = resolve_or_compile_model_url(path, &error); - - MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - config.computeUnits = resolve_compute_units_for_model(path, false); - CACTUS_LOG_INFO("npu", - "ANEEncoder compute units: " - << compute_units_to_string(config.computeUnits) - << " for " << [path UTF8String]); - - _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; - if (_model) { - _modelDescription = _model.modelDescription; - } - if (error) { - CACTUS_LOG_ERROR("npu", "ANE model load failed: " << [[error localizedDescription] UTF8String]); - } - } - return self; -} - -- (NSArray*)getInputShape { - if (!_modelDescription) return @[]; - - NSString* inputName = _cachedInputName; - if (!inputName) { - inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { - return inputDesc.multiArrayConstraint.shape; - } - - return @[]; -} - -- (NSArray*)getOutputShape { - if (!_modelDescription) return @[]; - - NSString* outputName = _cachedOutputName; - if (!outputName) { - outputName = _modelDescription.outputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - return outputDesc.multiArrayConstraint.shape; - } - - return @[]; -} - -- (BOOL)preallocateBuffersWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_model) return NO; - - NSError* error = nil; - - MLMultiArrayDataType inputDataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.multiArrayConstraint) { - inputDataType = inputDesc.multiArrayConstraint.dataType; - } - - _cachedInputArray = [[MLMultiArray alloc] - initWithShape:shape - dataType:inputDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE preallocate input array failed: " << [[error localizedDescription] UTF8String]); - return NO; - } - - _cachedShape = [shape copy]; - _cachedInputName = [inputName copy]; - _cachedOutputName = outputName ? [outputName copy] - : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - - CACTUS_LOG_DEBUG("npu", - "[CactusANE] prealloc input name=" - << [_cachedInputName UTF8String] - << " shape=" << [[_cachedInputArray.shape description] UTF8String] - << " strides=" << [[_cachedInputArray.strides description] UTF8String] - << " dtype=" << (long)_cachedInputArray.dataType); - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[_cachedOutputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* outputShape = outputDesc.multiArrayConstraint.shape; - MLMultiArrayDataType outputDataType = outputDesc.multiArrayConstraint.dataType; - - _cachedOutputArray = [[MLMultiArray alloc] - initWithShape:outputShape - dataType:outputDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE preallocate output array failed: " << [[error localizedDescription] UTF8String]); - return NO; - } - - _cachedOutputShape = [outputShape copy]; - _cachedOutputSize = 1; - for (NSNumber* dim in outputShape) { - _cachedOutputSize *= [dim unsignedIntegerValue]; - } - - CACTUS_LOG_DEBUG("npu", - "[CactusANE] prealloc output name=" - << [_cachedOutputName UTF8String] - << " shape=" << [[_cachedOutputArray.shape description] UTF8String] - << " strides=" << [[_cachedOutputArray.strides description] UTF8String] - << " dtype=" << (long)_cachedOutputArray.dataType); - - _predictionOptions = [[MLPredictionOptions alloc] init]; - _hasOutputBackings = NO; - if (_predictionOptions && _cachedOutputArray && _cachedOutputName) { - _predictionOptions.outputBackings = - @{ _cachedOutputName: _cachedOutputArray }; - _hasOutputBackings = YES; - } - } - - return YES; -} - -- (BOOL)canUseCachedBufferWithInput:(NSString*)inputName - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_cachedInputArray || !_cachedShape) return NO; - if (![_cachedInputName isEqualToString:inputName]) return NO; - if (_cachedShape.count != shape.count) return NO; - - for (NSUInteger i = 0; i < shape.count; i++) { - if (![_cachedShape[i] isEqualToNumber:shape[i]]) return NO; - } - - if (outputName && outputName.length > 0 && ![_cachedOutputName isEqualToString:outputName]) { - return NO; - } - - return YES; -} - -- (MLMultiArray*)predictWithInput:(NSString*)inputName - data:(const __fp16*)data - shape:(NSArray*)shape - outputName:(NSString*)outputName { - if (!_model) return nil; - - NSError* error = nil; - MLMultiArray* inputArray = nil; - - MLMultiArrayDataType expectedDataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.multiArrayConstraint) { - expectedDataType = inputDesc.multiArrayConstraint.dataType; - } - - BOOL useCached = (expectedDataType == MLMultiArrayDataTypeFloat16) && - [self canUseCachedBufferWithInput:inputName shape:shape outputName:outputName]; - - if (useCached) { - inputArray = _cachedInputArray; - } else { - inputArray = [[MLMultiArray alloc] - initWithShape:shape - dataType:expectedDataType - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE create input array failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - } - - NSUInteger totalElements = 1; - for (NSNumber* dim in shape) { - totalElements *= [dim unsignedIntegerValue]; - } - - if (expectedDataType == MLMultiArrayDataTypeFloat16) { - __fp16* inputPtr = (__fp16*)inputArray.dataPointer; - memcpy(inputPtr, data, totalElements * sizeof(__fp16)); - } else { - float* inputPtr = (float*)inputArray.dataPointer; - for (NSUInteger i = 0; i < totalElements; i++) { - inputPtr[i] = (float)data[i]; - } - } - - MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:inputArray]; - NSDictionary* inputDict = @{inputName: inputFeature}; - id inputProvider = [[MLDictionaryFeatureProvider alloc] - initWithDictionary:inputDict - error:&error]; - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE create feature provider failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - - id outputProvider = nil; - - if (useCached && _hasOutputBackings && _predictionOptions) { - outputProvider = [_model predictionFromFeatures:inputProvider - options:_predictionOptions - error:&error]; - } else { - outputProvider = [_model predictionFromFeatures:inputProvider error:&error]; - } - - if (error) { - CACTUS_LOG_ERROR("npu", "ANE prediction failed: " << [[error localizedDescription] UTF8String]); - return nil; - } - - NSString* outName = outputName; - if (!outName || outName.length == 0) { - outName = useCached ? _cachedOutputName - : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - } - - MLFeatureValue* outputFeature = [outputProvider featureValueForName:outName]; - static bool logged_output_feature_layout = false; - if (!logged_output_feature_layout && outputFeature && outputFeature.multiArrayValue) { - MLMultiArray* outArr = outputFeature.multiArrayValue; - CACTUS_LOG_DEBUG("npu", - "[CactusANE] output feature name=" - << [outName UTF8String] - << " shape=" << [[outArr.shape description] UTF8String] - << " strides=" << [[outArr.strides description] UTF8String] - << " dtype=" << (long)outArr.dataType); - logged_output_feature_layout = true; - } - return outputFeature.multiArrayValue; -} - -static size_t copyMLArrayToFP16(MLMultiArray* array, __fp16* output) { - size_t count = array.count; - if (!array || !output || count == 0) return 0; - - std::vector dims; - std::vector strides; - const size_t rank = array.shape.count; - bool have_layout = rank == array.strides.count && rank > 0; - if (have_layout) { - dims.resize(rank); - strides.resize(rank); - for (size_t i = 0; i < rank; ++i) { - NSInteger d = [array.shape[i] integerValue]; - NSInteger s = [array.strides[i] integerValue]; - if (d <= 0 || s < 0) { - have_layout = false; - break; - } - dims[i] = static_cast(d); - strides[i] = static_cast(s); - } - } - - if (!have_layout) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - if (output != (__fp16*)array.dataPointer) { - memcpy(output, array.dataPointer, count * sizeof(__fp16)); - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; - } - return count; - } - - auto is_contiguous = [&]() -> bool { - if (!have_layout) return false; - size_t expected = 1; - for (size_t i = rank; i-- > 0;) { - if (dims[i] > 1 && strides[i] != expected) return false; - expected *= dims[i]; - } - return true; - }; - - if (false && is_contiguous()) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - if (output != (__fp16*)array.dataPointer) { - memcpy(output, array.dataPointer, count * sizeof(__fp16)); - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; i++) output[i] = (__fp16)src[i]; - } - return count; - } - - std::vector idx(rank, 0); - if (array.dataType == MLMultiArrayDataTypeFloat16) { - const __fp16* src = (const __fp16*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - output[i] = src[offset]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } else { - const float* src = (const float*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - output[i] = (__fp16)src[offset]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } - return count; -} - -static void copyFP16ToMLArray(const __fp16* data, size_t count, MLMultiArray* array) { - if (!array || !data || count == 0) return; - - std::vector dims; - std::vector strides; - const size_t rank = array.shape.count; - bool have_layout = rank == array.strides.count && rank > 0; - if (have_layout) { - dims.resize(rank); - strides.resize(rank); - for (size_t i = 0; i < rank; ++i) { - NSInteger d = [array.shape[i] integerValue]; - NSInteger s = [array.strides[i] integerValue]; - if (d <= 0 || s < 0) { - have_layout = false; - break; - } - dims[i] = static_cast(d); - strides[i] = static_cast(s); - } - } - - if (!have_layout) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - memcpy(array.dataPointer, data, count * sizeof(__fp16)); - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; - } - return; - } - - auto is_contiguous = [&]() -> bool { - if (!have_layout) return false; - size_t expected = 1; - for (size_t i = rank; i-- > 0;) { - if (dims[i] > 1 && strides[i] != expected) return false; - expected *= dims[i]; - } - return true; - }; - - if (false && is_contiguous()) { - if (array.dataType == MLMultiArrayDataTypeFloat16) { - memcpy(array.dataPointer, data, count * sizeof(__fp16)); - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; i++) dst[i] = (float)data[i]; - } - return; - } - - std::vector idx(rank, 0); - if (array.dataType == MLMultiArrayDataTypeFloat16) { - __fp16* dst = (__fp16*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - dst[offset] = data[i]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } else { - float* dst = (float*)array.dataPointer; - for (size_t i = 0; i < count; ++i) { - size_t offset = 0; - for (size_t d = 0; d < rank; ++d) { - offset += idx[d] * strides[d]; - } - dst[offset] = (float)data[i]; - - for (size_t d = rank; d-- > 0;) { - idx[d]++; - if (idx[d] < dims[d]) break; - idx[d] = 0; - } - } - } -} - -- (BOOL)preallocateMultiInputBuffersWithOutputName:(NSString*)outputName { - if (!_model || !_modelDescription) return NO; - - NSError* error = nil; - _cachedMultiInputArrays = [NSMutableDictionary new]; - - for (NSString* inputName in _modelDescription.inputDescriptionsByName) { - MLFeatureDescription* desc = _modelDescription.inputDescriptionsByName[inputName]; - if (desc.type != MLFeatureTypeMultiArray) continue; - - MLMultiArray* array = [[MLMultiArray alloc] - initWithShape:desc.multiArrayConstraint.shape - dataType:desc.multiArrayConstraint.dataType - error:&error]; - if (!array || error) return NO; - _cachedMultiInputArrays[inputName] = array; - } - - NSString* outName = (outputName && outputName.length > 0) - ? outputName : _modelDescription.outputDescriptionsByName.allKeys.firstObject; - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - _cachedMultiOutputArray = [[MLMultiArray alloc] - initWithShape:outputDesc.multiArrayConstraint.shape - dataType:outputDesc.multiArrayConstraint.dataType - error:&error]; - if (!_cachedMultiOutputArray || error) return NO; - - _multiPredictionOptions = [[MLPredictionOptions alloc] init]; - _hasMultiOutputBackings = NO; - } - - _multiInputPreallocated = YES; - return YES; -} - -- (size_t)predictMultiInputWithDict:(NSMutableDictionary*)inputDict - outputData:(__fp16*)output - outputName:(NSString*)outputName { - if (!_model) return 0; - - NSError* error = nil; - MLDictionaryFeatureProvider* provider = - [[MLDictionaryFeatureProvider alloc] initWithDictionary:inputDict error:&error]; - if (!provider || error) return 0; - - id result = nil; - if (_hasMultiOutputBackings && _multiPredictionOptions) { - result = [_model predictionFromFeatures:provider options:_multiPredictionOptions error:&error]; - } else { - result = [_model predictionFromFeatures:provider error:&error]; - } - if (!result || error) return 0; - - static bool logged_multi_layout = false; - if (!logged_multi_layout) { - for (NSString* key in inputDict) { - MLFeatureValue* inFeature = inputDict[key]; - if (inFeature && inFeature.multiArrayValue) { - MLMultiArray* arr = inFeature.multiArrayValue; - } - } - } - - MLFeatureValue* outFeature = [result featureValueForName:outputName]; - if (!outFeature || !outFeature.multiArrayValue) return 0; - if (!logged_multi_layout) { - MLMultiArray* outArr = outFeature.multiArrayValue; - logged_multi_layout = true; - } - return copyMLArrayToFP16(outFeature.multiArrayValue, output); -} - -@end - -namespace cactus { -namespace npu { - -static bool g_npu_enabled = true; - -ANEEncoder::ANEEncoder() : impl_(nullptr) {} - -ANEEncoder::~ANEEncoder() { - if (impl_) { - (void)(__bridge_transfer CactusANEImpl*)impl_; - impl_ = nullptr; - } -} - -ANEEncoder::ANEEncoder(ANEEncoder&& other) noexcept : impl_(other.impl_) { - other.impl_ = nullptr; -} - -ANEEncoder& ANEEncoder::operator=(ANEEncoder&& other) noexcept { - if (this != &other) { - if (impl_) { - (void)(__bridge_transfer CactusANEImpl*)impl_; - } - impl_ = other.impl_; - other.impl_ = nullptr; - } - return *this; -} - -bool ANEEncoder::load(const std::string& model_path) { - @autoreleasepool { - CACTUS_LOG_INFO("npu", "ANEEncoder loading model: " << model_path); - NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; - - if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { - CACTUS_LOG_WARN("npu", "not using NPU"); - return false; - } - - CactusANEImpl* impl = [[CactusANEImpl alloc] initWithModelPath:path]; - - if (impl && impl.model) { - impl_ = (__bridge_retained void*)impl; - CACTUS_LOG_INFO("npu", "ANEEncoder model loaded successfully: " << model_path); - return true; - } - CACTUS_LOG_ERROR("npu", "ANEEncoder model load failed: " << model_path); - return false; - } -} - -bool ANEEncoder::preallocate(const std::vector& input_shape, - const std::string& input_name, - const std::string& output_name) { - if (!impl_) return false; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSMutableArray* shapeArray = [NSMutableArray array]; - for (int dim : input_shape) { - [shapeArray addObject:@(dim)]; - } - - NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; - NSString* outName = output_name.empty() - ? nil - : [NSString stringWithUTF8String:output_name.c_str()]; - - return [impl preallocateBuffersWithInput:inName shape:shapeArray outputName:outName]; - } -} - -size_t ANEEncoder::encode(const __fp16* input, - __fp16* output, - const std::vector& shape, - const std::string& input_name, - const std::string& output_name) { - if (!impl_ || !input || !output) return 0; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSArray* shapeArray = impl.cachedShape; - bool shapeMatches = (shapeArray && shapeArray.count == shape.size()); - if (shapeMatches) { - for (size_t i = 0; i < shape.size(); ++i) { - if ([shapeArray[i] intValue] != shape[i]) { - shapeMatches = false; - break; - } - } - } - - if (!shapeMatches) { - NSMutableArray* newShapeArray = [NSMutableArray arrayWithCapacity:shape.size()]; - for (int dim : shape) { - [newShapeArray addObject:@(dim)]; - } - shapeArray = newShapeArray; - } - - NSString* inName = impl.cachedInputName; - if (!inName) { - inName = [NSString stringWithUTF8String:input_name.c_str()]; - } - NSString* outName = impl.cachedOutputName; - if (!outName && !output_name.empty()) { - outName = [NSString stringWithUTF8String:output_name.c_str()]; - } - - MLMultiArray* mlOutput = [impl predictWithInput:inName - data:input - shape:shapeArray - outputName:outName]; - - if (mlOutput) { - return copyMLArrayToFP16(mlOutput, output); - } - } - - return 0; -} - -bool ANEEncoder::is_available() const { - if (!impl_) return false; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - return impl.model != nil; -} - -std::vector ANEEncoder::get_input_shape() const { - std::vector result; - if (!impl_) return result; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - NSArray* shape = [impl getInputShape]; - for (NSNumber* dim in shape) { - result.push_back([dim intValue]); - } - return result; -} - -std::vector ANEEncoder::get_output_shape() const { - std::vector result; - if (!impl_) return result; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - NSArray* shape = [impl getOutputShape]; - for (NSNumber* dim in shape) { - result.push_back([dim intValue]); - } - return result; -} - -__fp16* ANEEncoder::get_output_buffer() { - if (!impl_) return nullptr; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - if (!impl.hasOutputBackings || !impl.cachedOutputArray) return nullptr; - return (__fp16*)impl.cachedOutputArray.dataPointer; -} - -size_t ANEEncoder::get_output_buffer_size() const { - if (!impl_) return 0; - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - if (!impl.hasOutputBackings || !impl.cachedOutputArray) return 0; - return impl.cachedOutputSize; -} - -size_t ANEEncoder::encode_multimodal_input( - const std::vector& inputs, - __fp16* output, - const std::string& output_name) { - if (!impl_ || !output || inputs.empty()) return 0; - - @autoreleasepool { - CactusANEImpl* impl = (__bridge CactusANEImpl*)impl_; - - NSString* outName = output_name.empty() - ? impl.modelDescription.outputDescriptionsByName.allKeys.firstObject - : [NSString stringWithUTF8String:output_name.c_str()]; - - if (!impl.multiInputPreallocated) { - [impl preallocateMultiInputBuffersWithOutputName:outName]; - } - - NSMutableDictionary* inputDict = [NSMutableDictionary new]; - - for (const auto& input : inputs) { - NSString* nsName = [NSString stringWithUTF8String:input.name.c_str()]; - - size_t total = 1; - for (int dim : input.shape) total *= dim; - - MLMultiArray* array = impl.cachedMultiInputArrays[nsName]; - if (!array) { - MLMultiArrayDataType dataType = MLMultiArrayDataTypeFloat16; - MLFeatureDescription* desc = impl.modelDescription.inputDescriptionsByName[nsName]; - if (desc && desc.multiArrayConstraint) dataType = desc.multiArrayConstraint.dataType; - - NSMutableArray* shapeArray = [NSMutableArray arrayWithCapacity:input.shape.size()]; - for (int dim : input.shape) [shapeArray addObject:@(dim)]; - - NSError* arrayError = nil; - array = [[MLMultiArray alloc] initWithShape:shapeArray dataType:dataType error:&arrayError]; - if (!array || arrayError) return 0; - } - - copyFP16ToMLArray(input.data, total, array); - inputDict[nsName] = [MLFeatureValue featureValueWithMultiArray:array]; - } - - return [impl predictMultiInputWithDict:inputDict outputData:output outputName:outName]; - } -} - -std::unique_ptr create_encoder() { - return std::make_unique(); -} - -bool is_npu_available() { - CACTUS_LOG_INFO("npu", "is_npu_available: " << (g_npu_enabled ? "true" : "false")); - return g_npu_enabled; -} - -} // namespace npu -} // namespace cactus - -@interface CactusANEPrefillImpl : NSObject - -@property (nonatomic, strong) MLModel* model; -@property (nonatomic, strong) MLModelDescription* modelDescription; -@property (nonatomic, assign) int chunkSize; -@property (nonatomic, assign) int hiddenDim; -@property (nonatomic, assign) int numLayers; -@property (nonatomic, assign) int numKvHeads; -@property (nonatomic, assign) int headDim; - -@property (nonatomic, strong) MLMultiArray* cachedInputArray; -@property (nonatomic, strong) MLMultiArray* cachedOffsetArray; -@property (nonatomic, strong) NSMutableDictionary* cachedOutputArrays; -@property (nonatomic, strong) MLPredictionOptions* predictionOptions; -@property (nonatomic, assign) BOOL hasOutputBackings; - -- (instancetype)initWithModelPath:(NSString*)path; -- (void)preallocateBuffers; -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset; -- (MLMultiArray*)getOutputArray:(NSString*)name; -@end - -@implementation CactusANEPrefillImpl - -- (instancetype)initWithModelPath:(NSString*)path { - self = [super init]; - if (self) { - NSError* error = nil; - NSURL* modelURL = resolve_or_compile_model_url(path, &error); - - MLModelConfiguration* config = [[MLModelConfiguration alloc] init]; - config.computeUnits = resolve_compute_units_for_model(path, true); - CACTUS_LOG_INFO("npu", - "ANEPrefill compute units: " - << compute_units_to_string(config.computeUnits) - << " for " << [path UTF8String]); - - _model = [MLModel modelWithContentsOfURL:modelURL configuration:config error:&error]; - if (_model) { - _modelDescription = _model.modelDescription; - [self inferModelDimensions]; - [self preallocateBuffers]; - } - if (error) { - CACTUS_LOG_ERROR("npu", "ANE prefill model load failed: " << [[error localizedDescription] UTF8String]); - } - } - return self; -} - -- (void)inferModelDimensions { - if (!_modelDescription) return; - - NSString* inputName = _modelDescription.inputDescriptionsByName.allKeys.firstObject; - MLFeatureDescription* inputDesc = _modelDescription.inputDescriptionsByName[inputName]; - if (inputDesc && inputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = inputDesc.multiArrayConstraint.shape; - if (shape.count >= 2) { - _chunkSize = [shape[0] intValue]; - _hiddenDim = [shape[1] intValue]; - } - } - - int maxLayerIdx = -1; - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - if ([outputName hasPrefix:@"k_"]) { - int layerIdx = [[outputName substringFromIndex:2] intValue]; - maxLayerIdx = MAX(maxLayerIdx, layerIdx); - - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* shape = outputDesc.multiArrayConstraint.shape; - if (shape.count >= 3) { - _numKvHeads = [shape[1] intValue]; - _headDim = [shape[2] intValue]; - } - } - } - } - _numLayers = maxLayerIdx + 1; -} - -- (void)preallocateBuffers { - if (!_model || !_modelDescription || _chunkSize == 0 || _hiddenDim == 0) return; - - NSError* error = nil; - - NSArray* inputShape = @[@(_chunkSize), @(_hiddenDim)]; - _cachedInputArray = [[MLMultiArray alloc] initWithShape:inputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (error) { - _cachedInputArray = nil; - return; - } - - if (_modelDescription.inputDescriptionsByName[@"offset"] != nil) { - _cachedOffsetArray = [[MLMultiArray alloc] initWithShape:@[@1] - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (error) { - _cachedOffsetArray = nil; - } - } - - _cachedOutputArrays = [NSMutableDictionary dictionary]; - NSMutableDictionary* outputBackings = [NSMutableDictionary dictionary]; - - for (NSString* outputName in _modelDescription.outputDescriptionsByName.allKeys) { - MLFeatureDescription* outputDesc = _modelDescription.outputDescriptionsByName[outputName]; - if (outputDesc && outputDesc.type == MLFeatureTypeMultiArray) { - NSArray* outputShape = outputDesc.multiArrayConstraint.shape; - - MLMultiArray* outputArray = [[MLMultiArray alloc] initWithShape:outputShape - dataType:MLMultiArrayDataTypeFloat16 - error:&error]; - if (!error && outputArray) { - _cachedOutputArrays[outputName] = outputArray; - outputBackings[outputName] = outputArray; - } - } - } - - _predictionOptions = [[MLPredictionOptions alloc] init]; - _hasOutputBackings = NO; - if (@available(macOS 14.0, iOS 17.0, *)) { - _predictionOptions.outputBackings = outputBackings; - _hasOutputBackings = YES; - } -} - -- (BOOL)predictDirectWithInput:(NSString*)inputName - data:(const __fp16*)data - offset:(int)offset { - if (!_model || !_cachedInputArray) return NO; - - NSError* error = nil; - - NSUInteger totalElements = (NSUInteger)(_chunkSize * _hiddenDim); - __fp16* inputPtr = (__fp16*)_cachedInputArray.dataPointer; - memcpy(inputPtr, data, totalElements * sizeof(__fp16)); - - MLFeatureValue* inputFeature = [MLFeatureValue featureValueWithMultiArray:_cachedInputArray]; - NSMutableDictionary* inputDict = [NSMutableDictionary dictionaryWithObject:inputFeature forKey:inputName]; - - if (_cachedOffsetArray) { - ((__fp16*)_cachedOffsetArray.dataPointer)[0] = (__fp16)offset; - MLFeatureValue* offsetFeature = [MLFeatureValue featureValueWithMultiArray:_cachedOffsetArray]; - inputDict[@"offset"] = offsetFeature; - } - - id inputProvider = [[MLDictionaryFeatureProvider alloc] - initWithDictionary:inputDict - error:&error]; - if (error) return NO; - - id outputProvider = [_model predictionFromFeatures:inputProvider - options:_predictionOptions - error:&error]; - if (error || !outputProvider) return NO; - - if (!_hasOutputBackings) { - for (NSString* outputName in _cachedOutputArrays) { - MLFeatureValue* outFeature = [outputProvider featureValueForName:outputName]; - if (!outFeature || !outFeature.multiArrayValue) return NO; - - MLMultiArray* dst = _cachedOutputArrays[outputName]; - if (!dst) return NO; - - size_t copied = copyMLArrayToFP16(outFeature.multiArrayValue, (__fp16*)dst.dataPointer); - if (copied == 0) return NO; - } - } - - return YES; -} - -- (MLMultiArray*)getOutputArray:(NSString*)name { - return _cachedOutputArrays[name]; -} - -@end - -namespace cactus { -namespace npu { - -ANEPrefill::ANEPrefill() : impl_(nullptr) {} - -ANEPrefill::~ANEPrefill() { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - impl_ = nullptr; - } -} - -ANEPrefill::ANEPrefill(ANEPrefill&& other) noexcept : impl_(other.impl_), - chunk_size_(other.chunk_size_), hidden_dim_(other.hidden_dim_), - num_layers_(other.num_layers_), num_kv_heads_(other.num_kv_heads_), - head_dim_(other.head_dim_) { - other.impl_ = nullptr; -} - -ANEPrefill& ANEPrefill::operator=(ANEPrefill&& other) noexcept { - if (this != &other) { - if (impl_) { - (void)(__bridge_transfer CactusANEPrefillImpl*)impl_; - } - impl_ = other.impl_; - chunk_size_ = other.chunk_size_; - hidden_dim_ = other.hidden_dim_; - num_layers_ = other.num_layers_; - num_kv_heads_ = other.num_kv_heads_; - head_dim_ = other.head_dim_; - other.impl_ = nullptr; - } - return *this; -} - -bool ANEPrefill::load(const std::string& model_path) { - @autoreleasepool { - CACTUS_LOG_INFO("npu", "ANEPrefill loading model: " << model_path); - NSString* path = [NSString stringWithUTF8String:model_path.c_str()]; - - if (![[NSFileManager defaultManager] fileExistsAtPath:path]) { - CACTUS_LOG_DEBUG("npu", "ANEPrefill model file not found: " << model_path); - return false; - } - - CactusANEPrefillImpl* impl = [[CactusANEPrefillImpl alloc] initWithModelPath:path]; - - if (impl && impl.model) { - impl_ = (__bridge_retained void*)impl; - chunk_size_ = impl.chunkSize; - hidden_dim_ = impl.hiddenDim; - num_layers_ = impl.numLayers; - num_kv_heads_ = impl.numKvHeads; - head_dim_ = impl.headDim; - CACTUS_LOG_INFO("npu", "ANEPrefill model loaded successfully: " << model_path); - return true; - } - CACTUS_LOG_ERROR("npu", "ANEPrefill model load failed: " << model_path); - return false; - } -} - -bool ANEPrefill::is_available() const { - if (!impl_) return false; - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - return impl.model != nil; -} - -int ANEPrefill::get_chunk_size() const { return chunk_size_; } -int ANEPrefill::get_hidden_dim() const { return hidden_dim_; } -int ANEPrefill::get_num_layers() const { return num_layers_; } -int ANEPrefill::get_num_kv_heads() const { return num_kv_heads_; } -int ANEPrefill::get_head_dim() const { return head_dim_; } - -NPUPrefillDirectResult ANEPrefill::prefill_chunk_direct( - const std::vector<__fp16>& embeddings, - int position_offset, - const std::string& input_name) { - - NPUPrefillDirectResult result; - result.valid = false; - result.hidden = {nullptr, 0}; - result.k_caches.resize(num_layers_, {nullptr, 0}); - result.v_caches.resize(num_layers_, {nullptr, 0}); - - if (!impl_) return result; - - @autoreleasepool { - CactusANEPrefillImpl* impl = (__bridge CactusANEPrefillImpl*)impl_; - - NSString* inName = [NSString stringWithUTF8String:input_name.c_str()]; - - BOOL success = [impl predictDirectWithInput:inName - data:embeddings.data() - offset:position_offset]; - - if (!success) return result; - - MLMultiArray* hiddenArray = [impl getOutputArray:@"hidden"]; - if (hiddenArray) { - result.hidden.data = (__fp16*)hiddenArray.dataPointer; - result.hidden.count = hiddenArray.count; - } - - for (int layer = 0; layer < num_layers_; layer++) { - NSString* kName = [NSString stringWithFormat:@"k_%d", layer]; - NSString* vName = [NSString stringWithFormat:@"v_%d", layer]; - - MLMultiArray* kArray = [impl getOutputArray:kName]; - MLMultiArray* vArray = [impl getOutputArray:vName]; - - if (kArray) { - result.k_caches[layer].data = (__fp16*)kArray.dataPointer; - result.k_caches[layer].count = kArray.count; - } - if (vArray) { - result.v_caches[layer].data = (__fp16*)vArray.dataPointer; - result.v_caches[layer].count = vArray.count; - } - } - - result.valid = true; - } - - return result; -} - -std::unique_ptr create_prefill() { - return std::make_unique(); -} - -} // namespace npu -} // namespace cactus - -#else // !CACTUS_HAS_ANE - -namespace cactus { -namespace npu { - -std::unique_ptr create_encoder() { - return nullptr; -} - -bool is_npu_available() { - return false; -} - -std::unique_ptr create_prefill() { - return nullptr; -} - -} // namespace npu -} // namespace cactus - -#endif // CACTUS_HAS_ANE diff --git a/cactus/telemetry/telemetry.cpp b/cactus/telemetry/telemetry.cpp deleted file mode 100644 index 7fdbcfc6c..000000000 --- a/cactus/telemetry/telemetry.cpp +++ /dev/null @@ -1,1547 +0,0 @@ -#include "telemetry/telemetry.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(__APPLE__) -#include -#include -#endif -#include -#include -#include -#include -#include - -namespace cactus { -namespace telemetry { - -enum EventType { INIT = 0, COMPLETION = 1, EMBEDDING = 2, TRANSCRIPTION = 3, STREAM_TRANSCRIBE = 4 }; - -struct Event { - EventType type; - char model[128]; - bool success; - bool cloud_handoff; - double ttft_ms; - double prefill_tps; - double decode_tps; - double tps; - double response_time_ms; - double confidence; - double ram_usage_mb; - int tokens; - int prefill_tokens; - int decode_tokens; - double session_ttft_ms; - double session_tps; - double session_time_ms; - int session_tokens; - char message[256]; - char error[256]; - char function_calls[1024]; - std::chrono::system_clock::time_point timestamp; -}; -static bool enabled = false; -static int inference_active = 0; -static bool shutdown_called = false; -static bool atexit_registered = false; -static bool curl_initialized = false; -static bool cloud_disabled = false; -static std::string supabase_url = "https://vlqqczxwyaodtcdmdmlw.supabase.co"; -static std::string supabase_key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InZscXFjenh3eWFvZHRjZG1kbWx3Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3NTE1MTg2MzIsImV4cCI6MjA2NzA5NDYzMn0.nBzqGuK9j6RZ6mOPWU2boAC_5H9XDs-fPpo5P3WZYbI"; -static std::string device_id; -static std::string project_id; -static std::string cloud_key; -static std::string project_scope; -static std::string device_model; -static std::string device_os; -static std::string device_os_version; -static std::string device_brand; -static std::string cactus_version; -#ifndef CACTUS_DEFAULT_FRAMEWORK -#define CACTUS_DEFAULT_FRAMEWORK "cpp" -#endif -static std::string framework = CACTUS_DEFAULT_FRAMEWORK; -static std::string custom_cache_location; -static std::string app_id; -static std::string device_registered_file; -static std::string project_registered_file; -static bool device_registered = false; -static bool project_registered = false; -static bool ids_ready = false; -static bool in_stream_mode = false; - -static std::mutex telemetry_mutex; -static std::condition_variable telemetry_lifecycle_cv; - -enum class TelemetryLifecycleState { - Stopped, - Running, - ShuttingDown, -}; - -static TelemetryLifecycleState lifecycle_state = TelemetryLifecycleState::Stopped; - -static bool can_record_event_locked() { - return enabled && ids_ready && lifecycle_state == TelemetryLifecycleState::Running; -} - -struct CloudSendResult { - bool payload_ok = false; - bool project_registered_ok = false; - bool device_registered_ok = false; -}; - -struct CloudConfigurationStateSnapshot { - std::string supabase_url; - std::string supabase_key; - std::string device_id; - std::string project_id; - std::string cloud_key; - std::string project_scope; - std::string framework; - std::string cactus_version; - std::string device_model; - std::string device_os; - std::string device_os_version; - std::string device_brand; - std::string app_id; - std::string device_registered_file; - std::string project_registered_file; - bool enabled = false; - bool cloud_disabled = false; - bool device_registered = false; - bool project_registered = false; -}; - -static std::string new_uuid(); -static std::string format_timestamp(const std::chrono::system_clock::time_point& tp); -static std::string model_basename(const char* model_path) { - if (!model_path) return {}; - std::string m(model_path); - size_t pos = m.find_last_of("/\\"); - if (pos != std::string::npos && pos + 1 < m.size()) { - m = m.substr(pos + 1); - } - return m; -} - -// Forward declarations for helpers used before definition -static std::string event_type_to_string(EventType t); -static bool event_type_from_string(const std::string& s, EventType& t); -static bool extract_string_field(const std::string& line, const std::string& key, std::string& out); -static bool extract_json_field(const std::string& line, const std::string& key, std::string& out); -static bool extract_bool_field(const std::string& line, const std::string& key, bool& out); -static bool extract_double_field(const std::string& line, const std::string& key, double& out); -static bool extract_int_field(const std::string& line, const std::string& key, int& out); -static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out); -static void process_events(const std::vector& fresh_events); -static std::string get_telemetry_dir_locked(); -static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked(); - -class TelemetryDispatcher { -public: - TelemetryDispatcher() = default; - - static TelemetryDispatcher& instance() { - static std::once_flag once; - static TelemetryDispatcher* singleton = nullptr; - std::call_once(once, [] { - singleton = new TelemetryDispatcher(); - }); - return *singleton; - } - - void start() { - std::lock_guard lock(queue_mutex_); - if (running_) return; - stop_ = false; - processing_ = false; - running_ = true; - worker_thread_ = std::thread([this] { worker_loop(); }); - } - - bool enqueue(const Event& event) { - std::lock_guard lock(queue_mutex_); - if (!running_) return false; - pending_events_.push_back(event); - enqueue_seq_ += 1; - queue_cv_.notify_one(); - return true; - } - - void flush() { - uint64_t target = 0; - { - std::unique_lock lock(queue_mutex_); - if (running_) { - target = enqueue_seq_; - queue_cv_.notify_one(); - flush_cv_.wait(lock, [this, target] { - return processed_seq_ >= target && !processing_; - }); - } - } - process_with_io_lock({}); - } - - void stop() { - { - std::lock_guard lock(queue_mutex_); - stop_ = true; - queue_cv_.notify_all(); - } - - if (worker_thread_.joinable()) { - worker_thread_.join(); - } - - std::lock_guard lock(queue_mutex_); - running_ = false; - stop_ = false; - processing_ = false; - pending_events_.clear(); - enqueue_seq_ = 0; - processed_seq_ = 0; - } - -private: - void process_with_io_lock(const std::vector& batch) { - std::lock_guard io_guard(io_mutex_); - process_events(batch); - } - - void worker_loop() { - while (true) { - std::vector batch; - uint64_t processed_count = 0; - { - std::unique_lock lock(queue_mutex_); - queue_cv_.wait(lock, [this] { - return stop_ || !pending_events_.empty(); - }); - if (stop_ && pending_events_.empty()) { - break; - } - batch.assign(pending_events_.begin(), pending_events_.end()); - processed_count = static_cast(batch.size()); - pending_events_.clear(); - processing_ = true; - } - - process_with_io_lock(batch); - - { - std::lock_guard lock(queue_mutex_); - processed_seq_ += processed_count; - processing_ = false; - flush_cv_.notify_all(); - } - } - - std::lock_guard lock(queue_mutex_); - running_ = false; - processing_ = false; - flush_cv_.notify_all(); - } - std::mutex queue_mutex_; - std::mutex io_mutex_; - std::condition_variable queue_cv_; - std::condition_variable flush_cv_; - std::deque pending_events_; - std::thread worker_thread_; - bool stop_ = false; - bool running_ = false; - bool processing_ = false; - uint64_t enqueue_seq_ = 0; - uint64_t processed_seq_ = 0; -}; - -static void mkdir_p(const std::string& path) { - if (path.empty()) return; - - std::string current; - for (size_t i = 0; i < path.size(); ++i) { - current += path[i]; - if (path[i] == '/' && i > 0) { - mkdir(current.c_str(), 0755); - } - } - mkdir(path.c_str(), 0755); -} - -static std::string get_telemetry_dir_locked() { - const std::string& cache_location = custom_cache_location; - if (!cache_location.empty()) { - mkdir_p(cache_location); - return cache_location; - } - -#if defined(__ANDROID__) - { - std::ifstream f("/proc/self/cmdline"); - if (f.is_open()) { - std::string pkg; - std::getline(f, pkg, '\0'); - size_t colon = pkg.find(':'); - if (colon != std::string::npos) pkg = pkg.substr(0, colon); - if (!pkg.empty()) { - uid_t uid = getuid(); - int user_id = static_cast((uid - 10000) / 100000); - std::string dir = "/data/user/" + std::to_string(user_id) + "/" + pkg + "/cache/cactus/telemetry"; - mkdir_p(dir); - struct stat st; - if (stat(dir.c_str(), &st) == 0) return dir; - dir = "/data/data/" + pkg + "/cache/cactus/telemetry"; - mkdir_p(dir); - if (stat(dir.c_str(), &st) == 0) return dir; - } - } - } -#endif - - const char* home = getenv("HOME"); - if (!home) home = "/tmp"; - std::string dir = std::string(home) + "/Library/Caches/cactus/telemetry"; - mkdir((std::string(home) + "/Library").c_str(), 0755); - mkdir((std::string(home) + "/Library/Caches").c_str(), 0755); - mkdir((std::string(home) + "/Library/Caches/cactus").c_str(), 0755); - mkdir(dir.c_str(), 0755); - return dir; -} - -static std::string cloud_api_key_cache_file_locked() { - return get_telemetry_dir_locked() + "/cloud_api_key"; -} - -static std::string load_or_create_id(const std::string& file) { - std::ifstream in(file); - if (in.is_open()) { - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line; - } - } - std::string id = new_uuid(); - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << id; - } - return id; -} - -static bool load_registered_flag(const std::string& file) { - std::ifstream in(file); - if (!in.is_open()) return false; - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line[0] == '1'; - } - return false; -} - -static void persist_registered_flag(const std::string& file) { - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << "1"; - } -} - -static std::string sysctl_string(const char* key) { -#if defined(__APPLE__) - size_t size = 0; - if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) return {}; - std::string out(size, '\0'); - if (sysctlbyname(key, out.data(), &size, nullptr, 0) != 0) return {}; - if (!out.empty() && out.back() == '\0') out.pop_back(); - return out; -#else - (void)key; - return {}; -#endif -} - -static Event make_event(EventType type, const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { - Event e; - e.type = type; - e.success = success; - e.cloud_handoff = false; - e.ttft_ms = ttft_ms; - e.prefill_tps = 0.0; - e.decode_tps = tps; - e.tps = tps; - e.response_time_ms = response_time_ms; - e.confidence = 0.0; - e.ram_usage_mb = 0.0; - e.tokens = tokens; - e.prefill_tokens = 0; - e.decode_tokens = tokens; - e.session_ttft_ms = 0.0; - e.session_tps = 0.0; - e.session_time_ms = 0.0; - e.session_tokens = 0; - e.timestamp = std::chrono::system_clock::now(); - std::memset(e.model, 0, sizeof(e.model)); - std::memset(e.message, 0, sizeof(e.message)); - std::memset(e.error, 0, sizeof(e.error)); - std::memset(e.function_calls, 0, sizeof(e.function_calls)); - std::string safe_model = model_basename(model); - if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); - if (message) std::strncpy(e.message, message, sizeof(e.message)-1); - return e; -} - -static Event make_event_extended(EventType type, const char* model, const CompletionMetrics& metrics) { - Event e; - e.type = type; - e.success = metrics.success; - e.cloud_handoff = metrics.cloud_handoff; - e.ttft_ms = metrics.ttft_ms; - e.prefill_tps = metrics.prefill_tps; - e.decode_tps = metrics.decode_tps; - e.tps = metrics.decode_tps; - e.response_time_ms = metrics.response_time_ms; - e.confidence = metrics.confidence; - e.ram_usage_mb = metrics.ram_usage_mb; - e.tokens = static_cast(metrics.prefill_tokens + metrics.decode_tokens); - e.prefill_tokens = static_cast(metrics.prefill_tokens); - e.decode_tokens = static_cast(metrics.decode_tokens); - e.session_ttft_ms = 0.0; - e.session_tps = 0.0; - e.session_time_ms = 0.0; - e.session_tokens = 0; - e.timestamp = std::chrono::system_clock::now(); - std::memset(e.model, 0, sizeof(e.model)); - std::memset(e.message, 0, sizeof(e.message)); - std::memset(e.error, 0, sizeof(e.error)); - std::memset(e.function_calls, 0, sizeof(e.function_calls)); - std::string safe_model = model_basename(model); - if (!safe_model.empty()) std::strncpy(e.model, safe_model.c_str(), sizeof(e.model)-1); - if (!metrics.success && metrics.error_message) std::strncpy(e.error, metrics.error_message, sizeof(e.error)-1); - if (metrics.function_calls_json) std::strncpy(e.function_calls, metrics.function_calls_json, sizeof(e.function_calls)-1); - return e; -} - -static bool parse_event_line(const std::string& line, Event& out) { - std::string type_str; - if (!extract_string_field(line, "event_type", type_str)) return false; - EventType et; - if (!event_type_from_string(type_str, et)) return false; - - std::string model; - extract_string_field(line, "model", model); - bool success = false; - extract_bool_field(line, "success", success); - bool cloud_handoff = false; - extract_bool_field(line, "cloud_handoff", cloud_handoff); - double ttft = 0.0; - extract_double_field(line, "ttft", ttft); - double prefill_tps = 0.0; - extract_double_field(line, "prefill_tps", prefill_tps); - double decode_tps = 0.0; - extract_double_field(line, "decode_tps", decode_tps); - double tps = 0.0; - extract_double_field(line, "tps", tps); - if (decode_tps == 0.0 && tps != 0.0) decode_tps = tps; - double response_time = 0.0; - extract_double_field(line, "response_time", response_time); - double confidence = 0.0; - extract_double_field(line, "confidence", confidence); - double ram_usage_mb = 0.0; - extract_double_field(line, "ram_usage_mb", ram_usage_mb); - int tokens = 0; - extract_int_field(line, "tokens", tokens); - int prefill_tokens = 0; - extract_int_field(line, "prefill_tokens", prefill_tokens); - int decode_tokens = 0; - extract_int_field(line, "decode_tokens", decode_tokens); - double session_ttft_ms = 0.0; - extract_double_field(line, "session_ttft", session_ttft_ms); - double session_tps = 0.0; - extract_double_field(line, "session_tps", session_tps); - double session_time_ms = 0.0; - extract_double_field(line, "session_time_ms", session_time_ms); - int session_tokens = 0; - extract_int_field(line, "session_tokens", session_tokens); - std::string message; - extract_string_field(line, "message", message); - std::string error; - extract_string_field(line, "error", error); - std::string function_calls; - extract_json_field(line, "function_calls", function_calls); - double ts_ms = 0.0; - if (!extract_double_field_raw(line, "ts_ms", ts_ms)) { - extract_double_field(line, "ts_ms", ts_ms); - } - auto ts_point = std::chrono::system_clock::now(); - if (ts_ms > 0.0) { - ts_point = std::chrono::system_clock::time_point(std::chrono::milliseconds(static_cast(ts_ms))); - } - - out = make_event(et, - model.empty() ? nullptr : model.c_str(), - success, - ttft, - tps, - response_time, - tokens, - message.empty() ? nullptr : message.c_str()); - out.timestamp = ts_point; - out.cloud_handoff = cloud_handoff; - out.prefill_tps = prefill_tps; - out.decode_tps = decode_tps; - out.confidence = confidence; - out.ram_usage_mb = ram_usage_mb; - out.prefill_tokens = prefill_tokens; - out.decode_tokens = decode_tokens; - out.session_ttft_ms = session_ttft_ms; - out.session_tps = session_tps; - out.session_time_ms = session_time_ms; - out.session_tokens = session_tokens; - if (!error.empty()) std::strncpy(out.error, error.c_str(), sizeof(out.error)-1); - if (!function_calls.empty()) std::strncpy(out.function_calls, function_calls.c_str(), sizeof(out.function_calls)-1); - return true; -} - -static std::string event_type_to_string(EventType t) { - switch (t) { - case INIT: return "init"; - case COMPLETION: return "completion"; - case EMBEDDING: return "embedding"; - case TRANSCRIPTION: return "transcription"; - case STREAM_TRANSCRIBE: return "stream_transcribe"; - default: return "unknown"; - } -} - -static bool event_type_from_string(const std::string& s, EventType& t) { - if (s == "init") { t = INIT; return true; } - if (s == "completion") { t = COMPLETION; return true; } - if (s == "embedding") { t = EMBEDDING; return true; } - if (s == "transcription") { t = TRANSCRIPTION; return true; } - if (s == "stream_transcribe") { t = STREAM_TRANSCRIBE; return true; } - return false; -} - -static bool extract_string_field(const std::string& line, const std::string& key, std::string& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - if (line[pos] == '"') { - pos++; - size_t end = line.find('"', pos); - if (end == std::string::npos) return false; - out = line.substr(pos, end - pos); - return true; - } - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - out = line.substr(pos, end - pos); - if (out == "null") out.clear(); - return true; -} - -static bool extract_json_field(const std::string& line, const std::string& key, std::string& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - - char start_char = line[pos]; - if (start_char == '[' || start_char == '{') { - char end_char = (start_char == '[') ? ']' : '}'; - int depth = 0; - size_t end = pos; - bool in_string = false; - bool escape_next = false; - - for (; end < line.size(); end++) { - if (escape_next) { - escape_next = false; - continue; - } - if (line[end] == '\\') { - escape_next = true; - continue; - } - if (line[end] == '"') { - in_string = !in_string; - continue; - } - if (in_string) continue; - - if (line[end] == start_char) { - depth++; - } else if (line[end] == end_char) { - depth--; - if (depth == 0) { - out = line.substr(pos, end - pos + 1); - return true; - } - } - } - return false; - } - - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - out = line.substr(pos, end - pos); - if (out == "null") out.clear(); - return true; -} - -static bool extract_bool_field(const std::string& line, const std::string& key, bool& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - if (raw == "true") { out = true; return true; } - if (raw == "false") { out = false; return true; } - return false; -} - -static bool extract_double_field(const std::string& line, const std::string& key, double& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - try { - out = std::stod(raw); - return true; - } catch (...) { - return false; - } -} - -// Helper used when the field is already numeric (no quotes) in the log cache. -static bool extract_double_field_raw(const std::string& line, const std::string& key, double& out) { - std::string needle = "\"" + key + "\":"; - size_t pos = line.find(needle); - if (pos == std::string::npos) return false; - pos += needle.size(); - while (pos < line.size() && line[pos] == ' ') pos++; - if (pos >= line.size()) return false; - size_t end = line.find_first_of(",}", pos); - if (end == std::string::npos) end = line.size(); - std::string raw = line.substr(pos, end - pos); - try { - out = std::stod(raw); - return true; - } catch (...) { - return false; - } -} - -static bool extract_int_field(const std::string& line, const std::string& key, int& out) { - std::string raw; - if (!extract_string_field(line, key, raw)) return false; - try { - out = std::stoi(raw); - return true; - } catch (...) { - return false; - } -} - -static std::string new_uuid() { - static thread_local std::mt19937_64 rng(std::random_device{}()); - uint64_t a = rng(); - uint64_t b = rng(); - a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; - b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; - std::ostringstream oss; - oss << std::hex << std::setfill('0'); - oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); - oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); - oss << "-" << std::setw(4) << (a & 0xffffULL); - oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); - oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); - return oss.str(); -} - -static uint64_t fnv1a_64(const std::string& s) { - uint64_t h = 14695981039346656037ULL; - for (unsigned char c : s) { h ^= c; h *= 1099511628211ULL; } - return h; -} - -static std::string uuid_from_string(const std::string& input) { - uint64_t a = fnv1a_64(input); - uint64_t b = fnv1a_64(input + "\xff"); - a = (a & 0xffffffffffff0fffULL) | 0x0000000000004000ULL; - b = (b & 0x3fffffffffffffffULL) | 0x8000000000000000ULL; - std::ostringstream oss; - oss << std::hex << std::setfill('0'); - oss << std::setw(8) << ((a >> 32) & 0xffffffffULL); - oss << "-" << std::setw(4) << ((a >> 16) & 0xffffULL); - oss << "-" << std::setw(4) << (a & 0xffffULL); - oss << "-" << std::setw(4) << ((b >> 48) & 0xffffULL); - oss << "-" << std::setw(12) << (b & 0xffffffffffffULL); - return oss.str(); -} - -static std::string read_git_remote_url() { - const char* cwd_env = std::getenv("PWD"); - if (!cwd_env) return ""; - std::string dir = cwd_env; - while (!dir.empty()) { - std::ifstream f(dir + "/.git/config"); - if (f.is_open()) { - std::string line; - bool in_origin = false; - while (std::getline(f, line)) { - if (line.find("[remote \"origin\"]") != std::string::npos) { - in_origin = true; - } else if (in_origin) { - size_t pos = line.find("url = "); - if (pos != std::string::npos) return line.substr(pos + 6); - if (!line.empty() && line[0] == '[') break; - } - } - return ""; - } - size_t pos = dir.rfind('/'); - if (pos == 0 || pos == std::string::npos) break; - dir = dir.substr(0, pos); - } - return ""; -} - -#if defined(__APPLE__) -static std::string detect_app_id() { - CFBundleRef bundle = CFBundleGetMainBundle(); - if (!bundle) return ""; - CFStringRef id = (CFStringRef)CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleIdentifier")); - if (!id) return ""; - char buf[256]; - return CFStringGetCString(id, buf, sizeof(buf), kCFStringEncodingUTF8) ? buf : ""; -} -#elif defined(__linux__) || defined(__ANDROID__) -static std::string detect_app_id() { - std::ifstream f("/proc/self/cmdline"); - if (!f.is_open()) return ""; - std::string s; - std::getline(f, s, '\0'); - size_t colon = s.find(':'); - if (colon != std::string::npos) s = s.substr(0, colon); - return s; -} -#else -static std::string detect_app_id() { return ""; } -#endif - -static std::string extract_android_package_from_path(const std::string& path) { - for (const char* prefix : {"/data/user/", "/data/user_de/", "/data/data/"}) { - size_t pos = path.find(prefix); - if (pos == std::string::npos) continue; - size_t start = pos + strlen(prefix); - if (strcmp(prefix, "/data/data/") != 0) { - start = path.find('/', start); - if (start == std::string::npos) continue; - start++; - } - size_t end = path.find('/', start); - if (end == std::string::npos) end = path.size(); - std::string pkg = path.substr(start, end - start); - if (!pkg.empty()) return pkg; - } - return ""; -} - -static std::string format_timestamp(const std::chrono::system_clock::time_point& tp) { - using namespace std::chrono; - auto secs = time_point_cast(tp); - auto ms = duration_cast(tp - secs).count(); - std::time_t t = system_clock::to_time_t(secs); - std::tm tm{}; -#if defined(_WIN32) - gmtime_s(&tm, &t); -#else - gmtime_r(&t, &tm); -#endif - char buf[32]; - std::snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03lldZ", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, tm.tm_sec, - static_cast(ms)); - return std::string(buf); -} - -[[maybe_unused]] static bool is_valid_uuid(const std::string& s) { - // Simple check used only for validation when accepting user-provided IDs. - if (s.size() != 36) return false; - for (size_t i = 0; i < s.size(); ++i) { - char c = s[i]; - if (i == 8 || i == 13 || i == 18 || i == 23) { - if (c != '-') return false; - } else { - bool hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - if (!hex) return false; - } - } - return true; -} - -static std::string escape_json_string(const char* str) { - if (!str) return ""; - std::string result; - result.reserve(std::strlen(str)); - for (const char* p = str; *p; ++p) { - switch (*p) { - case '"': result += "\\\""; break; - case '\\': result += "\\\\"; break; - case '\b': result += "\\b"; break; - case '\f': result += "\\f"; break; - case '\n': result += "\\n"; break; - case '\r': result += "\\r"; break; - case '\t': result += "\\t"; break; - default: - if (static_cast(*p) < 0x20) { - char buf[7]; - snprintf(buf, sizeof(buf), "\\u%04x", static_cast(*p)); - result += buf; - } else { - result += *p; - } - break; - } - } - return result; -} - -static void collect_device_info() { - struct utsname u; - if (uname(&u) == 0) { - const std::string sysname = u.sysname; - device_os = (sysname == "Darwin") ? "macos" : sysname; - device_model = u.machine; - device_os_version = u.release; - device_brand = "unknown"; -#if defined(__APPLE__) - // Prefer user-facing macOS info over Darwin kernel version. - std::string hw_model = sysctl_string("hw.model"); - std::string os_product = sysctl_string("kern.osproductversion"); - if (!hw_model.empty()) device_model = hw_model; - if (!os_product.empty()) device_os_version = os_product; - device_brand = "apple"; -#elif defined(__ANDROID__) - device_brand = "android"; -#endif - } -} - -static bool parse_version_line(const std::string& line) { - size_t start = line.find_first_not_of(" \t\r\n"); - size_t end = line.find_last_not_of(" \t\r\n"); - if (start != std::string::npos && end != std::string::npos) { - cactus_version = line.substr(start, end - start + 1); - return true; - } - return false; -} - -static void read_cactus_version() { - const char* version_paths[] = { - "CACTUS_VERSION", - "../CACTUS_VERSION", - "../../CACTUS_VERSION", - "../../../CACTUS_VERSION" - }; - - for (const char* path : version_paths) { - std::ifstream file(path); - if (file.is_open()) { - std::string line; - if (std::getline(file, line) && parse_version_line(line)) return; - } - } - -#if defined(__APPLE__) - CFBundleRef bundle = CFBundleGetMainBundle(); - if (bundle) { - CFURLRef url = CFBundleCopyResourceURL(bundle, CFSTR("CACTUS_VERSION"), NULL, NULL); - if (url) { - char fspath[4096]; - if (CFURLGetFileSystemRepresentation(url, true, (UInt8*)fspath, sizeof(fspath))) { - std::ifstream file(fspath); - if (file.is_open()) { - std::string line; - if (std::getline(file, line) && parse_version_line(line)) { - CFRelease(url); - return; - } - } - } - CFRelease(url); - } - } -#endif - -#ifdef CACTUS_COMPILE_TIME_VERSION - cactus_version = CACTUS_COMPILE_TIME_VERSION; - return; -#endif - - cactus_version = ""; -} - -static void apply_curl_tls_trust(CURL* curl) { - if (!curl) return; - const char* ca_bundle = std::getenv("CACTUS_CA_BUNDLE"); - if (ca_bundle && ca_bundle[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAINFO, ca_bundle); - } -#if defined(__ANDROID__) - const char* ca_path = std::getenv("CACTUS_CA_PATH"); - if (ca_path && ca_path[0] != '\0') { - curl_easy_setopt(curl, CURLOPT_CAPATH, ca_path); - } else { - curl_easy_setopt(curl, CURLOPT_CAPATH, "/system/etc/security/cacerts"); - } -#endif -} - -static bool ensure_project_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { - if (snapshot.project_id.empty() || snapshot.project_registered) return true; - std::string url = snapshot.supabase_url + "/rest/v1/projects"; - std::ostringstream payload; - payload << "[{"; - payload << "\"project_key\":\"" << snapshot.project_id << "\""; - if (!snapshot.project_scope.empty()) { - payload << ",\"name\":\"" << snapshot.project_scope << "\""; - } - payload << "}]"; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - std::string body = payload.str(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); - return ok; -} - -static bool ensure_device_row_remote(CURL* curl, const CloudConfigurationStateSnapshot& snapshot) { - if (snapshot.device_registered) return true; - if (snapshot.device_id.empty()) return false; - std::string url = snapshot.supabase_url + "/rest/v1/devices"; - std::ostringstream payload; - payload << "[{"; - payload << "\"id\":\"" << snapshot.device_id << "\""; - payload << ",\"device_id\":\"" << snapshot.device_id << "\""; - if (!snapshot.device_model.empty()) payload << ",\"model\":\"" << snapshot.device_model << "\""; - if (!snapshot.device_os.empty()) payload << ",\"os\":\"" << snapshot.device_os << "\""; - if (!snapshot.device_os_version.empty()) payload << ",\"os_version\":\"" << snapshot.device_os_version << "\""; - if (!snapshot.device_brand.empty()) payload << ",\"brand\":\"" << snapshot.device_brand << "\""; - payload << "}]"; - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: resolution=ignore-duplicates"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - std::string body = payload.str(); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - bool ok = (res == CURLE_OK) && ((code >= 200 && code < 300) || code == 409); - return ok; -} - -static bool send_payload(CURL* curl, const std::string& url, const std::string& body, const CloudConfigurationStateSnapshot& snapshot) { - struct curl_slist* headers = nullptr; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string apikey_hdr = std::string("apikey: ") + snapshot.supabase_key; - std::string auth_hdr = std::string("Authorization: Bearer ") + snapshot.supabase_key; - headers = curl_slist_append(headers, apikey_hdr.c_str()); - headers = curl_slist_append(headers, auth_hdr.c_str()); - headers = curl_slist_append(headers, "Prefer: return=minimal"); - headers = curl_slist_append(headers, "Content-Profile: cactus"); - headers = curl_slist_append(headers, "Accept-Profile: cactus"); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size()); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); - apply_curl_tls_trust(curl); - CURLcode res = curl_easy_perform(curl); - long code = 0; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); - if (headers) curl_slist_free_all(headers); - return (res == CURLE_OK) && (code >= 200 && code < 300); -} - -static CloudSendResult send_batch_to_cloud(const std::vector& local, const CloudConfigurationStateSnapshot& snapshot) { - CloudSendResult result; - if (!snapshot.enabled) return result; - if (local.empty()) { - result.payload_ok = true; - return result; - } - if (snapshot.cloud_disabled) return result; - if (snapshot.device_id.empty()) return result; - CURL* curl = curl_easy_init(); - if (!curl) return result; - result.project_registered_ok = ensure_project_row_remote(curl, snapshot); - if (!snapshot.device_registered) { - result.device_registered_ok = ensure_device_row_remote(curl, snapshot); - } else { - result.device_registered_ok = true; - } - std::string url = snapshot.supabase_url + "/rest/v1/logs"; - std::ostringstream payload; - payload << "["; - for (size_t i = 0; i < local.size(); ++i) { - const Event& e = local[i]; - payload << "{"; - payload << "\"event_type\":\"" << event_type_to_string(e.type) << "\","; - payload << "\"model\":\"" << escape_json_string(e.model) << "\","; - payload << "\"success\":" << (e.success ? "true" : "false") << ","; - payload << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; - if (e.type == INIT || !e.success) { - payload << "\"ttft\":null,"; - payload << "\"prefill_tps\":null,"; - payload << "\"decode_tps\":null,"; - payload << "\"tps\":null,"; - } else { - payload << "\"ttft\":" << e.ttft_ms << ","; - payload << "\"prefill_tps\":" << e.prefill_tps << ","; - payload << "\"decode_tps\":" << e.decode_tps << ","; - payload << "\"tps\":" << e.tps << ","; - } - if (!e.success) { - payload << "\"response_time\":null,"; - payload << "\"ram_usage_mb\":null,"; - } else { - payload << "\"response_time\":" << e.response_time_ms << ","; - payload << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; - } - if (e.type == INIT || !e.success) { - payload << "\"confidence\":null,"; - payload << "\"tokens\":null,"; - payload << "\"prefill_tokens\":null,"; - payload << "\"decode_tokens\":null,"; - } else { - payload << "\"confidence\":" << e.confidence << ","; - payload << "\"tokens\":" << e.tokens << ","; - payload << "\"prefill_tokens\":" << e.prefill_tokens << ","; - payload << "\"decode_tokens\":" << e.decode_tokens << ","; - } - if (e.type == STREAM_TRANSCRIBE) { - payload << "\"session_ttft\":" << e.session_ttft_ms << ","; - payload << "\"session_tps\":" << e.session_tps << ","; - payload << "\"session_time_ms\":" << e.session_time_ms << ","; - payload << "\"session_tokens\":" << e.session_tokens << ","; - } else { - payload << "\"session_ttft\":null,"; - payload << "\"session_tps\":null,"; - payload << "\"session_time_ms\":null,"; - payload << "\"session_tokens\":null,"; - } - payload << "\"created_at\":\"" << format_timestamp(e.timestamp) << "\","; - if (!snapshot.project_id.empty()) { - payload << "\"project_id\":\"" << snapshot.project_id << "\","; - } - if (!snapshot.cloud_key.empty()) { - payload << "\"key_hash\":\"" << snapshot.cloud_key << "\","; - } - payload << "\"framework\":\"" << snapshot.framework << "\","; - if (!snapshot.cactus_version.empty()) { - payload << "\"framework_version\":\"" << snapshot.cactus_version << "\","; - } - payload << "\"device_id\":\"" << snapshot.device_id << "\""; - if (!snapshot.app_id.empty()) { - payload << ",\"app_id\":\"" << escape_json_string(snapshot.app_id.c_str()) << "\""; - } else { - payload << ",\"app_id\":null"; - } - if (e.message[0] != '\0') { - payload << ",\"message\":\"" << escape_json_string(e.message) << "\""; - } else { - payload << ",\"message\":null"; - } - if (e.error[0] != '\0') { - payload << ",\"error\":\"" << escape_json_string(e.error) << "\""; - } else { - payload << ",\"error\":null"; - } - payload << ",\"function_calls\":null"; - payload << "}"; - if (i + 1 < local.size()) payload << ","; - } - payload << "]"; - result.payload_ok = send_payload(curl, url, payload.str(), snapshot); - curl_easy_cleanup(curl); - return result; -} - -static void write_events_to_cache_in_dir(const std::vector& local, const std::string& dir) { - for (const auto &e : local) { - std::ostringstream oss; - oss << "{\"event_type\":\"" << event_type_to_string(e.type) << "\","; - oss << "\"model\":\"" << escape_json_string(e.model) << "\","; - oss << "\"success\":" << (e.success ? "true" : "false") << ","; - oss << "\"cloud_handoff\":" << (e.cloud_handoff ? "true" : "false") << ","; - if (e.type == INIT || !e.success) { - oss << "\"ttft\":null,"; - oss << "\"prefill_tps\":null,"; - oss << "\"decode_tps\":null,"; - oss << "\"tps\":null,"; - } else { - oss << "\"ttft\":" << e.ttft_ms << ","; - oss << "\"prefill_tps\":" << e.prefill_tps << ","; - oss << "\"decode_tps\":" << e.decode_tps << ","; - oss << "\"tps\":" << e.tps << ","; - } - if (!e.success) { - oss << "\"response_time\":null,"; - oss << "\"ram_usage_mb\":null,"; - } else { - oss << "\"response_time\":" << e.response_time_ms << ","; - oss << "\"ram_usage_mb\":" << e.ram_usage_mb << ","; - } - if (e.type == INIT || !e.success) { - oss << "\"confidence\":null,"; - oss << "\"tokens\":null,"; - oss << "\"prefill_tokens\":null,"; - oss << "\"decode_tokens\":null,"; - } else { - oss << "\"confidence\":" << e.confidence << ","; - oss << "\"tokens\":" << e.tokens << ","; - oss << "\"prefill_tokens\":" << e.prefill_tokens << ","; - oss << "\"decode_tokens\":" << e.decode_tokens << ","; - } - if (e.type == STREAM_TRANSCRIBE) { - oss << "\"session_ttft\":" << e.session_ttft_ms << ","; - oss << "\"session_tps\":" << e.session_tps << ","; - oss << "\"session_time_ms\":" << e.session_time_ms << ","; - oss << "\"session_tokens\":" << e.session_tokens; - } else { - oss << "\"session_ttft\":null,"; - oss << "\"session_tps\":null,"; - oss << "\"session_time_ms\":null,"; - oss << "\"session_tokens\":null"; - } - oss << ",\"ts_ms\":" << std::chrono::duration_cast(e.timestamp.time_since_epoch()).count(); - if (e.message[0] != '\0') { - oss << ",\"message\":\"" << escape_json_string(e.message) << "\""; - } else { - oss << ",\"message\":null"; - } - if (e.error[0] != '\0') { - oss << ",\"error\":\"" << escape_json_string(e.error) << "\""; - } else { - oss << ",\"error\":null"; - } - oss << ",\"function_calls\":null"; - if (!app_id.empty()) { - oss << ",\"app_id\":\"" << escape_json_string(app_id.c_str()) << "\""; - } else { - oss << ",\"app_id\":null"; - } - oss << "}"; - std::string file = dir + "/" + event_type_to_string(e.type) + ".log"; - std::ofstream out(file, std::ios::app); - if (out.is_open()) { - out << oss.str() << "\n"; - out.close(); - } - } -} - -static std::vector load_cached_events_in_dir(const std::string& dir) { - std::vector events; - DIR* d = opendir(dir.c_str()); - if (!d) return events; - struct dirent* ent; - while ((ent = readdir(d)) != nullptr) { - std::string name = ent->d_name; - if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; - std::ifstream in(dir + "/" + name); - std::string line; - while (std::getline(in, line)) { - Event e; - if (parse_event_line(line, e)) { - events.push_back(e); - } - } - } - closedir(d); - return events; -} - -static void clear_cache_files_in_dir(const std::string& dir) { - DIR* d = opendir(dir.c_str()); - if (!d) return; - struct dirent* ent; - while ((ent = readdir(d)) != nullptr) { - std::string name = ent->d_name; - if (name.size() < 5 || name.substr(name.size() - 4) != ".log") continue; - std::remove((dir + "/" + name).c_str()); - } - closedir(d); -} - -static CloudConfigurationStateSnapshot capture_cloud_configuration_state_snapshot_locked() { - CloudConfigurationStateSnapshot snapshot; - snapshot.supabase_url = supabase_url; - snapshot.supabase_key = supabase_key; - snapshot.device_id = device_id; - snapshot.project_id = project_id; - snapshot.cloud_key = cloud_key; - snapshot.project_scope = project_scope; - snapshot.framework = framework; - snapshot.cactus_version = cactus_version; - snapshot.device_model = device_model; - snapshot.device_os = device_os; - snapshot.device_os_version = device_os_version; - snapshot.device_brand = device_brand; - snapshot.app_id = app_id; - snapshot.device_registered_file = device_registered_file; - snapshot.project_registered_file = project_registered_file; - snapshot.enabled = enabled; - snapshot.cloud_disabled = cloud_disabled; - snapshot.device_registered = device_registered; - snapshot.project_registered = project_registered; - return snapshot; -} - -static void process_events(const std::vector& fresh_events) { - std::vector events; - CloudConfigurationStateSnapshot snapshot; - std::string telemetry_dir; - - { - std::lock_guard guard(telemetry_mutex); - telemetry_dir = get_telemetry_dir_locked(); - events = load_cached_events_in_dir(telemetry_dir); - if (!fresh_events.empty()) { - events.insert(events.end(), fresh_events.begin(), fresh_events.end()); - } - if (events.empty()) return; - snapshot = capture_cloud_configuration_state_snapshot_locked(); - } - - CloudSendResult send_result = send_batch_to_cloud(events, snapshot); - - { - std::lock_guard guard(telemetry_mutex); - if (send_result.project_registered_ok && !project_registered) { - if (!project_registered_file.empty()) { - persist_registered_flag(project_registered_file); - } - project_registered = true; - } - if (send_result.device_registered_ok && !device_registered) { - if (!device_registered_file.empty()) { - persist_registered_flag(device_registered_file); - } - device_registered = true; - } - - if (send_result.payload_ok) { - clear_cache_files_in_dir(telemetry_dir); - } else if (!fresh_events.empty()) { - write_events_to_cache_in_dir(fresh_events, telemetry_dir); - } - } -} - -void init(const char* project_id_param, const char* project_scope_param, const char* cloud_key_param) { - std::unique_lock lifecycle_guard(telemetry_mutex); - telemetry_lifecycle_cv.wait(lifecycle_guard, [] { - return lifecycle_state != TelemetryLifecycleState::ShuttingDown; - }); - - std::string scope = project_scope_param ? project_scope_param : "default"; - - bool cloud_disabled_from_env = false; - if (const char* env = std::getenv("CACTUS_NO_CLOUD_TELE")) { - if (env[0] != '\0' && !(env[0] == '0' && env[1] == '\0')) { - cloud_disabled_from_env = true; - } - } - const char* env_url = std::getenv("CACTUS_SUPABASE_URL"); - const char* env_key = std::getenv("CACTUS_SUPABASE_KEY"); - const char* env_project = std::getenv("CACTUS_PROJECT_ID"); - const char* env_cloud = std::getenv("CACTUS_CLOUD_KEY"); - - std::string dir = get_telemetry_dir_locked(); - - std::string device_file = dir + "/device_id"; - std::string device_flag_file = dir + "/device_registered"; - bool device_was_registered = load_registered_flag(device_flag_file); - std::string resolved_device_id = load_or_create_id(device_file); - - std::string resolved_project_id; - if (project_id_param && *project_id_param) { - resolved_project_id = project_id_param; - } else if (env_project && *env_project) { - resolved_project_id = env_project; - } else { - std::string remote_url = read_git_remote_url(); - resolved_project_id = remote_url.empty() ? resolved_device_id : uuid_from_string(remote_url); - } - - std::string project_flag_file = dir + "/project_reg_" + resolved_project_id.substr(0, 8); - bool project_was_registered = load_registered_flag(project_flag_file); - - std::string resolved_cloud_key; - if (cloud_key_param && *cloud_key_param) { - resolved_cloud_key = cloud_key_param; - } else if (env_cloud && *env_cloud) { - resolved_cloud_key = env_cloud; - } - - collect_device_info(); - if (cactus_version.empty()) read_cactus_version(); - - if (env_url && *env_url) supabase_url = env_url; - if (env_key && *env_key) supabase_key = env_key; - project_id = resolved_project_id; - project_scope = scope; - project_registered_file = project_flag_file; - cloud_key = resolved_cloud_key; - device_registered_file = device_flag_file; - device_id = resolved_device_id; - if (app_id.empty()) app_id = detect_app_id(); - - project_registered = project_was_registered; - device_registered = device_was_registered; - if (cloud_disabled_from_env) { - cloud_disabled = true; - } - - if (!curl_initialized) { - if (curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK) { - curl_initialized = true; - } - } - - if (!atexit_registered) { - atexit_registered = true; - std::atexit([](){ shutdown(); }); - } - - shutdown_called = false; - ids_ready = true; - enabled = true; - lifecycle_state = TelemetryLifecycleState::Running; - TelemetryDispatcher::instance().start(); -} - -void setEnabled(bool en) { - std::lock_guard guard(telemetry_mutex); - enabled = en; -} - -void setCloudDisabled(bool disabled) { - std::lock_guard guard(telemetry_mutex); - cloud_disabled = disabled; -} - -void setAppId(const char* id) { - std::lock_guard guard(telemetry_mutex); - if (id && id[0] != '\0') app_id = id; -} - -void setTelemetryEnvironment(const char* framework_str, const char* cache_location_str, const char* version_str) { - std::lock_guard guard(telemetry_mutex); - if (framework_str && framework_str[0] != '\0') { - framework = framework_str; - } - if (cache_location_str && cache_location_str[0] != '\0') { - custom_cache_location = cache_location_str; - if (app_id.empty()) { - std::string pkg = extract_android_package_from_path(custom_cache_location); - if (!pkg.empty()) app_id = pkg; - } - } - if (version_str && version_str[0] != '\0') { - cactus_version = version_str; - } -} - -void setCloudKey(const char* key) { - if (!key || key[0] == '\0') return; - std::lock_guard guard(telemetry_mutex); - cloud_key = key; -} - -void cacheCloudApiKey(const char* key) { - if (!key || key[0] == '\0') return; - std::lock_guard guard(telemetry_mutex); - const std::string file = cloud_api_key_cache_file_locked(); - std::ofstream out(file, std::ios::trunc); - if (out.is_open()) { - out << key; - } -} - -std::string loadCachedCloudApiKey() { - std::lock_guard guard(telemetry_mutex); - const std::string file = cloud_api_key_cache_file_locked(); - std::ifstream in(file); - if (!in.is_open()) return {}; - std::string line; - if (std::getline(in, line) && !line.empty()) { - return line; - } - return {}; -} - -void recordInit(const char* model, bool success, double response_time_ms, const char* message) { - double nan = std::numeric_limits::quiet_NaN(); - Event e = make_event(INIT, model, success, nan, nan, response_time_ms, 0, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordCompletion(const char* model, const CompletionMetrics& metrics) { - Event e = make_event_extended(COMPLETION, model, metrics); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message) { - Event e = make_event(COMPLETION, model, success, ttft_ms, tps, response_time_ms, tokens, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordEmbedding(const char* model, bool success, const char* message) { - Event e = make_event(EMBEDDING, model, success, 0.0, 0.0, 0.0, 0, message); - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message) { - std::lock_guard guard(telemetry_mutex); - if (in_stream_mode) return; - Event e = make_event(TRANSCRIPTION, model, success, ttft_ms, tps, response_time_ms, tokens, message); - e.ram_usage_mb = ram_usage_mb; - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message) { - Event e = make_event(STREAM_TRANSCRIBE, model, success, ttft_ms, tps, response_time_ms, tokens, message); - e.session_ttft_ms = session_ttft_ms; - e.session_tps = session_tps; - e.session_time_ms = session_time_ms; - e.session_tokens = session_tokens; - std::lock_guard guard(telemetry_mutex); - if (!can_record_event_locked()) return; - TelemetryDispatcher::instance().enqueue(e); -} - -void setStreamMode(bool in_stream) { - std::lock_guard guard(telemetry_mutex); - in_stream_mode = in_stream; -} - -bool isStreamMode() { - std::lock_guard guard(telemetry_mutex); - return in_stream_mode; -} - -void markInference(bool active) { - std::lock_guard guard(telemetry_mutex); - if (active) { - inference_active += 1; - } else if (inference_active > 0) { - inference_active -= 1; - } -} - -void flush() { - TelemetryDispatcher::instance().flush(); -} - -void shutdown() { - { - std::lock_guard lifecycle_guard(telemetry_mutex); - if (shutdown_called) { - return; - } - - shutdown_called = true; - lifecycle_state = TelemetryLifecycleState::ShuttingDown; - } - - flush(); - - { - std::lock_guard lifecycle_guard(telemetry_mutex); - enabled = false; - ids_ready = false; - } - - TelemetryDispatcher::instance().stop(); - - { - std::lock_guard lifecycle_guard(telemetry_mutex); - if (curl_initialized) { - curl_initialized = false; - curl_global_cleanup(); - } - lifecycle_state = TelemetryLifecycleState::Stopped; - telemetry_lifecycle_cv.notify_all(); - } -} - -} // namespace telemetry -} // namespace cactus diff --git a/cactus/telemetry/telemetry.h b/cactus/telemetry/telemetry.h deleted file mode 100644 index cd7b49f45..000000000 --- a/cactus/telemetry/telemetry.h +++ /dev/null @@ -1,44 +0,0 @@ -#pragma once -#include -#include - -namespace cactus { -namespace telemetry { - -struct CompletionMetrics { - bool success; - bool cloud_handoff; - double ttft_ms; - double prefill_tps; - double decode_tps; - double response_time_ms; - double confidence; - double ram_usage_mb; - size_t prefill_tokens; - size_t decode_tokens; - const char* error_message; - const char* function_calls_json; -}; - -void init(const char* project_id = nullptr, const char* project_scope = nullptr, const char* cloud_key = nullptr); -void setEnabled(bool enabled); -void setCloudDisabled(bool disabled); -void setAppId(const char* app_id); -void setTelemetryEnvironment(const char* framework, const char* cache_location, const char* version = nullptr); -void setCloudKey(const char* key); -void cacheCloudApiKey(const char* key); -std::string loadCachedCloudApiKey(); -void recordInit(const char* model, bool success, double response_time_ms, const char* message); -void recordCompletion(const char* model, const CompletionMetrics& metrics); -void recordCompletion(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, const char* message); -void recordEmbedding(const char* model, bool success, const char* message); -void recordTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double ram_usage_mb, const char* message); -void recordStreamTranscription(const char* model, bool success, double ttft_ms, double tps, double response_time_ms, int tokens, double session_ttft_ms, double session_tps, double session_time_ms, int session_tokens, const char* message); -void setStreamMode(bool in_stream); -bool isStreamMode(); -void markInference(bool active); -void flush(); -void shutdown(); - -} // namespace telemetry -} // namespace cactus diff --git a/docs/cactus_engine.md b/docs/cactus_engine.md index b140f4c64..dfcf47236 100644 --- a/docs/cactus_engine.md +++ b/docs/cactus_engine.md @@ -22,11 +22,15 @@ cactus download openai/whisper-small cactus auth ``` -`cactus download` fetches pre-converted CQ weights from +`cactus download` fetches a **pre-built runtime bundle** (CQ weights + serialized +graph + manifest) from [huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute) into -`weights/` and they can be loaded directly via `cactus_init()`. For -models without a pre-converted archive, use `cactus convert ` to build -them from source. +`transpiled/-cq[-]/`. Defaults to the generic CPU +bundle; pass `--platform apple` for the Apple Silicon variant. The result +can be loaded directly via `cactus_init()`. + +For models not on Cactus-Compute, build a bundle from source with +`cactus convert ` followed by `cactus transpile `. ## Types @@ -171,7 +175,7 @@ int cactus_complete( | `include_stop_sequences` | bool | false | Include stop sequence tokens in the response | | `force_tools` | bool | false | Constrain output to tool call format | | `tool_rag_top_k` | int | 2 | Select top-k relevant tools via Tool RAG (0 = disabled, use all tools) | -| `confidence_threshold` | float | 0.7 | Minimum confidence for local generation; triggers cloud_handoff when below | +| `confidence_threshold` | float | model-dependent | Minimum confidence for local generation; triggers cloud_handoff when below. Resolved in this order: `0.5` if the bundle ships a `handoff_probe.bin`; else the model's `default_cloud_handoff_threshold` (Gemma 4 = `0.92`); else `0.7`. | | `auto_handoff` | bool | true | Automatically attempt cloud handoff when confidence is low | | `cloud_timeout_ms` | int | 15000 | Timeout in milliseconds for cloud handoff requests | | `handoff_with_images` | bool | true | Allow cloud handoff for requests that include images | @@ -466,6 +470,24 @@ if (result >= 0) { } ``` +### `cactus_benchmark_tokens` +Runs a prefill + decode benchmark on the given prompt tokens and returns timing JSON +(prefill ms, decode ms, tokens/sec). Useful for measuring inference performance on +a specific model + device without running the full chat loop. + +```c +int cactus_benchmark_tokens( + cactus_model_t model, + const uint32_t* prompt_tokens, + size_t prompt_token_len, + size_t decode_token_len, + char* response_buffer, + size_t buffer_size +); +``` + +Returns the number of bytes written to `response_buffer` on success, `-1` on error. + ### `cactus_transcribe` Transcribes audio to text. Supports Whisper, Moonshine, and Parakeet models. Supports both file-based and buffer-based audio input. @@ -574,7 +596,7 @@ int result = cactus_transcribe(whisper, NULL, NULL, | Option | Type | Default | Description | |--------|------|---------|-------------| -| `max_tokens` | int | 448 | Maximum tokens to generate | +| `max_tokens` | int | auto | Maximum tokens to generate. When unset, scales with audio length (`audio_sec × 20` for Whisper, `× 30` for Parakeet), then clamped to the Whisper decoder ceiling of 448. | | `custom_vocabulary` | array | [] | List of words or phrases to bias the decoder toward. Useful for proper nouns, acronyms, medical terms, and domain-specific jargon. | | `vocabulary_boost` | float | 5.0 | Logit bias strength applied to tokens from `custom_vocabulary`. Clamped to 0.0–20.0. Higher values make the listed words more likely to appear. | @@ -595,70 +617,6 @@ if (result > 0) { } ``` -### `cactus_detect_language` -Detects the spoken language in an audio file or PCM buffer. - -```c -int cactus_detect_language( - cactus_model_t model, // Model handle (must be Whisper model) - const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer - char* response_buffer, // Buffer for response JSON - size_t buffer_size, // Size of response buffer - const char* options_json, // Optional options (can be NULL) - const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL if using file) - size_t pcm_buffer_size // Size of PCM buffer in bytes (must be even and >= 2) -); -``` - -**Returns:** Number of bytes written to response_buffer on success, negative value on error - -**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. Only Whisper models are supported; passing any other model type returns -1. - -**Options Format:** -```json -{ - "use_vad": true -} -``` - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `use_vad` | bool | true | Filter audio through VAD before language detection (requires model initialized with a VAD component) | - -**Response Format:** -```json -{ - "success": true, - "error": null, - "language": "en", - "language_token": "<|en|>", - "token_id": 50259, - "confidence": 0.9812, - "entropy": 0.0188, - "total_time_ms": 234.56, - "ram_usage_mb": 512.34 -} -``` - -- `language`: ISO 639-1 language code, or `"unknown"` if detection failed -- `language_token`: Raw token text emitted by the model for the language (e.g. `"<|en|>"`) -- `token_id`: Vocabulary token ID of the language token -- `confidence`: Detection confidence (0.0–1.0), derived as `1.0 - entropy` -- `entropy`: Normalized entropy of the sampled token -- `total_time_ms`: Total detection time in milliseconds -- `ram_usage_mb`: Current process RAM usage - -**Example:** -```c -cactus_model_t whisper = cactus_init("../../weights/whisper-small", NULL, false); - -char response[1024]; -int result = cactus_detect_language(whisper, "audio.wav", response, sizeof(response), NULL, NULL, 0); -if (result >= 0) { - printf("Detected language: %s\n", response); -} -``` - ### `cactus_embed` Generates text embeddings for semantic search, similarity, and RAG applications. @@ -1229,20 +1187,6 @@ int find_similar_image(cactus_model_t model, const char* query, } ``` -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `CACTUS_KV_WINDOW_SIZE` | 512 | Sliding window size for KV cache | -| `CACTUS_KV_SINK_SIZE` | 4 | Number of attention sink tokens to preserve | - -**Example:** -```bash -export CACTUS_KV_WINDOW_SIZE=1024 -export CACTUS_KV_SINK_SIZE=8 -./my_app -``` - ## Best Practices 1. **Always Check Return Values**: Functions return negative values on error @@ -1272,7 +1216,6 @@ Common error scenarios: 2. **Streaming for UX**: Use callbacks for responsive user interfaces 3. **Early Stopping**: Use `cactus_stop()` to avoid unnecessary generation 4. **Batch Embeddings**: When possible, process multiple texts in sequence without resetting -5. **KV Cache Tuning**: Adjust `CACTUS_KV_WINDOW_SIZE` based on your context needs ## Logging @@ -1344,5 +1287,5 @@ void cactus_telemetry_shutdown(void); - [Python Binding](/python/) — Python bindings for the Engine API - [Swift Binding](/bindings/swift/) — Swift bindings for iOS and macOS - [Kotlin Binding](/bindings/kotlin/) — Kotlin Multiplatform bindings -- [Flutter Binding](/flutter/) — Dart FFI bindings for mobile apps -- [Rust Binding](/rust/) — Rust FFI bindings via bindgen +- [Flutter Binding](/bindings/flutter/) — Dart FFI bindings for mobile apps +- [Rust Binding](/bindings/rust/) — Raw `extern "C"` FFI declarations diff --git a/docs/cactus_graph.md b/docs/cactus_graph.md index a3c6fc50e..cbf0f339e 100644 --- a/docs/cactus_graph.md +++ b/docs/cactus_graph.md @@ -197,18 +197,22 @@ size_t glu_out = graph.glu(input, axis); // Gated Linear Unit ``` #### Convolution Operations + +Each conv has two overloads: with and without bias. + ```cpp -// 1D convolutions -size_t conv1d_out = graph.conv1d(input, weight, has_bias, bias, stride); -size_t conv1d_k3_out = graph.conv1d_k3(input, weight, stride); -size_t conv1d_causal = graph.conv1d_causal(input, weight, kernel_size, dilation); -size_t conv1d_pointwise = graph.conv1d_pointwise(input, weight, has_bias, bias); -size_t conv1d_depthwise = graph.conv1d_same_depthwise_k9(input, weight, has_bias, bias); +// 1D convolutions (no bias / with bias overloads) +size_t conv1d_out = graph.conv1d(input, weight, stride); +size_t conv1d_with_bias = graph.conv1d(input, weight, bias, stride); +size_t conv1d_k3_out = graph.conv1d_k3(input, weight, stride); +size_t conv1d_causal = graph.conv1d_causal(input, weight, kernel_size, dilation); +size_t conv1d_pointwise = graph.conv1d_pointwise(input, weight); // or (input, weight, bias) +size_t conv1d_depthwise = graph.conv1d_same_depthwise_k9(input, weight); // or (input, weight, bias) // 2D convolutions -size_t conv2d_out = graph.conv2d_k3s2p1(input, weight, has_bias, bias); -size_t conv2d_dw = graph.conv2d_depthwise_k3s2p1(input, weight, has_bias, bias); -size_t conv2d_pw = graph.conv2d_pointwise_1x1(input, weight, has_bias, bias); +size_t conv2d_out = graph.conv2d_k3s2p1(input, weight); // or (input, weight, bias) +size_t conv2d_dw = graph.conv2d_depthwise_k3s2p1(input, weight); // or (input, weight, bias) +size_t conv2d_pw = graph.conv2d_pointwise_1x1(input, weight); // or (input, weight, bias) ``` #### Normalization @@ -226,11 +230,13 @@ size_t deltanet_prefill = graph.gated_deltanet_prefill(query, key, value, gate_l #### Mixture of Experts (MoE) ```cpp -size_t moe_gated = graph.moe_layer_gated(hidden, routing_probs, topk_indices, +// Gated SwiGLU MoE: pass three weight sets (w1=gate, w3=up, w2=down) +size_t moe_gated = graph.moe_layer(hidden, routing_probs, topk_indices, w1_weights, w3_weights, w2_weights, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor); -size_t moe_ungated = graph.moe_layer_ungated(hidden, routing_probs, topk_indices, +// Ungated MoE: pass two weight sets (w1=up, w2=down) and the activation explicitly +size_t moe_ungated = graph.moe_layer(hidden, routing_probs, topk_indices, w1_weights, w2_weights, num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, activation); ``` @@ -302,7 +308,12 @@ size_t interpolated = graph.bilinear_interpolation(pos_embeds, dst_height, dst_w #### Sampling ```cpp +// Defaults: temperature=0.6, top_p=0.95, top_k=20 size_t sampled = graph.sample(logits, temperature, top_p, top_k); + +// With per-token bias map and repetition penalty / min_p: +size_t sampled_ext = graph.sample_with_options( + logits, temperature, top_p, min_p, repetition_penalty, top_k, /*logit_bias=*/{}); ``` ## Advanced Features @@ -328,7 +339,6 @@ size_t result = graph.add(a, b); // {2,3} -> {2,2,3} ```cpp size_t int8_tensor = graph.input({4}, Precision::INT8); size_t fp16_tensor = graph.precision_cast(int8_tensor, Precision::FP16); -graph.set_quantization_scale(node_id, scale); ``` ### Graph Persistence @@ -359,15 +369,16 @@ loaded.execute(); GraphFile::save_node(graph, node_id, "output.bin"); ``` -#### Loading Nodes +#### Loading Graphs ```cpp -CactusGraph new_graph; -auto loaded = GraphFile::load_into_graph(new_graph, "output.bin"); -size_t node_id = loaded.node_id; -std::vector shape = loaded.shape; -Precision precision = loaded.precision; +CactusGraph loaded = GraphFile::load_graph("graph.cactus"); ``` +The `GraphFile` namespace currently exposes `save_graph`, `load_graph`, and +`save_node`. There is no per-node loader; reload a saved subgraph by +serializing the parent graph with `save_graph` and reading it back with +`load_graph`. + ### Graph Management #### Execution @@ -504,8 +515,7 @@ Add a builder method in `cactus-graph/src/builder.cpp` and its declaration in Follow the pattern of existing builder methods, e.g. for a new "relu" op: ```cpp size_t CactusGraph::relu(size_t input) { - OpParams params; - return add_node(OpType::RELU, {input}, params); + return add_node(OpType::RELU, {input}, {}); } ``` 3. ** Implement the op in the execution engine ** @@ -513,14 +523,14 @@ Implement the kernel or graph op code in the relevant file, usually in `cactus-k Register the new op in the dispatch table in `cactus-graph/src/execute.cpp` for the supported backends (CPU, NPU) 4. ** Export op in FFI bindings ** -- header: `cactus-engine/cactus_engine.h` -- implementation: `cactus-engine/src/graph_ffi.cpp` +- header: `cactus-graph/cactus_graph.h` (in the `extern "C"` block) +- implementation: `cactus-graph/src/graph_ffi.cpp` 5. ** Add python ctypes declaration ** Add `_lib.cactus_graph_my_new_op.argtypes/restype` in `python/cactus/bindings/cactus.py` 6. ** Add python graph wrapper ** -Add `Graph.my_new_op(...)` in `python/cactus/bindings/graph.py`, and optionally a Tensor +Add `Graph.my_new_op(...)` in `python/cactus/bindings/cactus.py`, and optionally a Tensor convenience method. 7. ** Add serialization schema entry if needed ** @@ -544,7 +554,7 @@ The syntax pattern there is: }}, ``` 8. ** Add test coverage ** -Add unit tests to `tests/test_graph.cpp` covering the native graph function, and +Add unit tests to `cactus-graph/tests/test_graph.cpp` covering the native graph function, and add python tests in `python/tests/test_graph.py` covering the Python API and end-to-end execution. and then support those fields in `write_field(...)` / `read_field(...)`. @@ -569,10 +579,10 @@ try { ```cpp CactusGraph graph; size_t x = graph.input({batch, dim}, Precision::FP16); -x = graph.linear(x, weight1, bias1); +x = graph.add(graph.matmul(x, weight1), bias1); x = graph.gelu(x); x = graph.layernorm(x, ln_weight1, ln_bias1); -x = graph.linear(x, weight2, bias2); +x = graph.add(graph.matmul(x, weight2), bias2); ``` ### Residual Connections diff --git a/docs/cactus_hybrid.md b/docs/cactus_hybrid.md index e304ca65a..1bc7f12b9 100644 --- a/docs/cactus_hybrid.md +++ b/docs/cactus_hybrid.md @@ -179,7 +179,7 @@ options = json.dumps({ "auto_handoff": True }) messages = json.dumps([{"role": "user", "content": "Explain quantum entanglement"}]) -result = json.loads(cactus_complete(model, messages, options, None, None)) +result = cactus_complete(model, messages, options, None, None) if result["cloud_handoff"]: print("Answered by cloud") diff --git a/docs/cactus_index.md b/docs/cactus_index.md index 455de9c7d..a0e669117 100644 --- a/docs/cactus_index.md +++ b/docs/cactus_index.md @@ -11,7 +11,7 @@ The Cactus Index provides a clean C FFI (Foreign Function Interface) for integra ## Getting Started The index uses memory-mapped files: -- `index.bin`: Embeddings (FP16) and metadata pointers +- `index.bin`: Embeddings (FP16) plus per-doc offsets into `data.bin` - `data.bin`: Document content and metadata (UTF-8) All embeddings are automatically normalized to unit length @@ -570,4 +570,4 @@ cactus_index_destroy(new_index); - [Python Binding](/python/) — Python bindings with vector index support - [Swift Binding](/bindings/swift/) — Swift vector index functions - [Kotlin Binding](/bindings/kotlin/) — Kotlin vector index functions -- [Flutter Binding](/flutter/) — Dart vector index functions +- [Flutter Binding](/bindings/flutter/) — Dart vector index functions diff --git a/docs/cactus_kernels.md b/docs/cactus_kernels.md index eb48b0e9f..de6602d32 100644 --- a/docs/cactus_kernels.md +++ b/docs/cactus_kernels.md @@ -31,7 +31,7 @@ void cactus_add_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t n); void cactus_subtract_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t n); void cactus_multiply_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t n); void cactus_divide_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t n); -void cactus_add_clipped_f16(const __fp16* a, const __fp16* b, __fp16* output, size_t n); +void cactus_add_f16_clipped(const __fp16* a, const __fp16* b, __fp16* output, size_t n); void cactus_add_scaled_f16(const __fp16* base, const __fp16* src, __fp16* output, size_t n, float scale); ``` @@ -156,7 +156,7 @@ void cactus_attention_f16( size_t batch_size, size_t seq_len, size_t kv_seq_len, size_t num_q_heads, size_t num_kv_heads, size_t head_dim, float scale, - const __fp16* mask = nullptr, + const __fp16* mask, size_t position_offset = 0, size_t window_size = 0, // 0 = full context, >0 = sliding window bool is_causal = true, @@ -431,30 +431,39 @@ void cactus_gaussian_topk_f16(const __fp16* input, __fp16* output, ``` cactus-kernels/ - cactus_kernels.h # public API (this file) - threading.h # thread pool utilities + cactus_kernels.h # public API (this file) libs/ - stb_image.h # vendored image loading - stb_image_resize2.h # vendored image resizing + stb_image.h # vendored image loading + stb_image_resize2.h # vendored image resizing src/ - arithmetic.cpp # element-wise ops, broadcast, reductions - matmul.cpp # FP16 GEMM - quant.cpp # CQ 1-4 bit GEMV/GEMM, dequantization - attention.cpp # attention kernels (FP16, hybrid INT8/FP16) - activations.cpp # relu, silu, gelu, sigmoid, tanh, glu, clamp - norm.cpp # rms_norm, layer_norm, batchnorm, softmax - conv.cpp # conv1d, conv2d variants - sample.cpp # top-k/top-p/min-p sampling - dsp.cpp # rfft, irfft, mel filter bank, spectrogram - image.cpp # image load/resize/normalize/patches - rope.cpp # rotary position embedding - transpose.cpp # 2D and N-D transpose - lstm.cpp # LSTM cell, BiLSTM sequence - deltanet.cpp # gated DeltaNet decode/prefill - wav.h # WAV file loading + 16kHz resampling + threading.h # thread pool utilities + wav.h # WAV file loading + 16 kHz resampling + attention.cpp # attention kernels (FP16) + attention_hybrid.cpp # hybrid INT8/FP16 attention + blas.cpp # BLAS-backed paths + conv.cpp # conv1d variants + conv2d.cpp # conv2d variants + dsp.cpp # rfft, irfft, mel filter bank, spectrogram, STFT + fused.cpp # fused op kernels + image.cpp # image load/resize/normalize/patches + lstm.cpp # LSTM cell, BiLSTM sequence + matmul.cpp # FP16/INT8 GEMM + nn.cpp # activations (relu/silu/gelu/...), glu, clamp + norms_rope.cpp # rms_norm, layer_norm, RoPE + quants.cpp # CQ 1-4 bit GEMV/GEMM, dequantization + reduce.cpp # reductions (sum/mean/var/min/max + axis variants) + scalar.cpp # scalar elementwise ops tests/ - test_kernels.cpp # 36 tests across 7 suites - test_utils.h # test runner, fp16 comparison helpers + test_utils.h # test runner, fp16 comparison helpers + test_attention.cpp + test_conv.cpp + test_dsp.cpp + test_elementwise.cpp + test_matmul.cpp + test_performance.cpp # perf benchmarks (run via `cactus test --suite performance`) + test_quant.cpp + test_reduce.cpp + android/ # on-device test runner for Android ``` ## See Also diff --git a/docs/cactus_quants.md b/docs/cactus_quants.md index cd6c868db..ac1079b5a 100644 --- a/docs/cactus_quants.md +++ b/docs/cactus_quants.md @@ -183,23 +183,30 @@ Three patterns are visible: ## Using CQ Weights -CQ weights are produced by `cactus convert` and consumed by the Cactus runtime: +CQ weights are produced by `cactus convert`. The runtime bundle (graph + manifest) +is then built from those weights with `cactus transpile`, and run via +`cactus run`: ```bash -# Convert at 4-bit (default) +# 1. Convert at 4-bit (default) cactus convert google/gemma-4-E2B-it ./gemma4-weights -# Convert at 2-bit +# 1'. Convert at 2-bit cactus convert google/gemma-4-E2B-it ./gemma4-weights --bits 2 -# Run -cactus run ./gemma4-weights +# 2. Build the runtime bundle from the CQ weights +cactus transpile google/gemma-4-E2B-it \ + --weights-dir ./gemma4-weights --artifact-dir ./gemma4-bundle + +# 3. Run (pass either a HF model id or a bundle path) +cactus run ./gemma4-bundle ``` -Pre-converted CQ weights for all supported models are available at -[huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute). +For models on huggingface.co/Cactus-Compute, you can skip the convert+transpile +steps entirely: `cactus run ` will fetch the pre-built bundle (or fall +back to local convert+transpile if no bundle is published), then run. -The CQ kernel implementation lives in `cactus-kernels/src/quant.cpp`. At inference, +The CQ matmul kernels live in `cactus-kernels/src/matmul.cpp`. At inference, the kernel applies the inverse Hadamard rotation to the FP16 activation (not the weight), then performs a fused codebook-lookup + norm-scale + matmul in a single NEON pass. diff --git a/docs/cactus_transpiler.md b/docs/cactus_transpiler.md index 90d18f0a3..5b0005c8b 100644 --- a/docs/cactus_transpiler.md +++ b/docs/cactus_transpiler.md @@ -55,7 +55,7 @@ cactus convert ./classifier.pt ./classifier-cactus --bits 4 cactus transpile ./classifier.pt --weights-dir ./classifier-cactus # Run the transpiled bundle -cactus run ./transpiled/classifier +cactus run ./classifier-cactus ``` That's it. The output is a self-contained bundle with memory-mapped weights that @@ -198,7 +198,7 @@ cactus convert google/gemma-4-E2B-it ./gemma4-weights --bits 4 cactus transpile google/gemma-4-E2B-it --weights-dir ./gemma4-weights # 3. Run -cactus run ./transpiled/google/gemma-4-E2B-it --prompt "Hello!" +cactus run ./gemma4-weights --prompt "Hello!" ``` ### Text Models @@ -240,26 +240,28 @@ cactus transpile nvidia/parakeet-tdt-0.6b-v3 \ ## Running Saved Bundles -Transpiled bundles are saved to `./transpiled//` by default. Run them later -without re-transpiling: +Transpiled bundles are saved to `./weights//` by default (alongside their CQ +weights). Run them later without re-transpiling: ```bash # Text -cactus run ./transpiled/Qwen_Qwen3-0.6B --prompt "Write a haiku" +cactus run ./weights/qwen3-0.6b --prompt "Write a haiku" # Multimodal -cactus run-transpiled ./transpiled/google/gemma-4-E2B-it \ - --image-file photo.jpg \ - --file speech.wav \ +cactus run ./weights/gemma-4-e2b-it \ + --image photo.jpg \ + --audio speech.wav \ --prompt "What do you see?" # Audio -cactus run-transpiled ./transpiled/nvidia/parakeet-tdt-0.6b-v3 \ - --file meeting.wav +cactus run ./weights/parakeet-tdt-0.6b-v3 \ + --audio meeting.wav ``` -`cactus run` auto-detects transpiled bundles by looking for a `manifest.json` in the -directory. +`cactus run` accepts either a HuggingFace model id (downloads the bundle) or a +local bundle directory path (detected by the presence of +`components/manifest.json`). The standalone `run-transpiled` subcommand was +removed — `cactus run` handles both cases. --- @@ -270,7 +272,7 @@ You can also drive the transpiler from Python: ```python from cactus.transpile.capture_pytorch import capture_model from cactus.transpile.lower import transpile_ir -from cactus.bindings.graph import Graph +from cactus.bindings.cactus import Graph import torch import numpy as np @@ -385,7 +387,7 @@ component. Use `loaded.reset()` between independent generations. A transpiled bundle looks like this: ``` -transpiled// +weights// raw_ir.json # IR before optimization (debugging) optimized_ir.json # IR after fusion passes (debugging) graph.cactus # serialized runtime graph @@ -420,34 +422,56 @@ cactus transpile [options] | Option | Description | |--------|-------------| -| `--weights-dir ` | Path to converted Cactus weights (required) | -| `--task ` | Force task type (`auto` by default) | -| `--prompt ` | Example text prompt for capture | -| `--image-file ` | Example image for multimodal capture | -| `--audio-file ` | Example audio for multimodal/speech capture | -| `--artifact-dir ` | Output directory (default: `./transpiled/`) | +| `--weights-dir ` | Path to converted CQ weights (default: `weights/`) | +| `--task ` | Force task type (default: `auto` — inferred from weights). Choices: `causal_lm_logits`, `multimodal_causal_lm_logits`, `ctc_logits`, `encoder_hidden_states`, `seq2seq_transcription`, `tdt_transcription` | +| `--prompt ` | Representative prompt for shape capture | +| `--image-file ` | Representative image (repeatable) | +| `--audio-file ` | Representative audio file | +| `--max-new-tokens ` | Generation room to preallocate for causal decode graphs | +| `--artifact-dir ` | Output directory (default: `weights/`) | | `--execute-after-transpile` | Run the graph after saving | | `--skip-reference-compare` | Skip PyTorch vs transpiled comparison | -| `--component-pipeline auto\|on\|off` | Control component splitting | +| `--component-pipeline auto\|on\|off` | Use split component graph transpilation when supported | +| `--components ` | Comma-separated component subset for component-pipeline models | +| `--trust-remote-code` | Allow HF remote code during the transpile phase | +| `--local-files-only` | Require HF files to already be local | +| `--allow-unconverted-weights` | Transpile against an unconverted source checkpoint | | `--no-fuse-rms-norm` | Disable RMSNorm fusion | | `--no-fuse-rope` | Disable RoPE fusion | | `--no-fuse-attention` | Disable attention fusion | -### `cactus run-transpiled` +NPU emission (CoreML `.mlpackage`s for Apple Silicon audio/vision encoders) is +implemented on the underlying `python -m cactus.transpile.hf_model` script +(`--npu`, `--npu-quantize`, `--npu-audio-quantize`, `--npu-vision-quantize`) +but is not yet exposed on the user-facing `cactus transpile` argparser. + +### `cactus run` ```bash -cactus run-transpiled [options] +cactus run [options] ``` +`model_id` may be a HuggingFace model id (downloads the matching bundle from +huggingface.co/Cactus-Compute) or a local path to a bundle directory. + | Option | Description | |--------|-------------| +| `--bits 1\|2\|3\|4` | CQ quantization bits when downloading (default: 4) | +| `--platform cpu\|apple` | Target accelerator (default: cpu — generic ARM) | +| `--token ` | HuggingFace token (gated models) | | `--prompt ` | Input prompt | | `--input-ids ` | Comma-separated token IDs | -| `--file ` | Audio file for speech bundles | -| `--image-file ` | Image file for multimodal bundles | -| `--weights-dir ` | Override weight directory | +| `--image ` | Image file for multimodal bundles | +| `--audio ` | Audio file for speech / audio bundles | | `--system ` | System prompt | -| `--thinking` | Enable chain-of-thought mode | +| `--thinking` | Enable thinking / reasoning mode | +| `--max-new-tokens ` | Max tokens to generate | +| `--result-json ` | Save bundle result payload to JSON | +| `--no-cloud-handoff` | Disable automatic cloud handoff | +| `--confidence-threshold ` | Confidence threshold below which to hand off to cloud | +| `--cloud-timeout-ms ` | Max wait time for cloud handoff | +| `--no-cloud-tele` | Disable cloud telemetry (write to cache only) | +| `--reconvert` | Force local convert+transpile fallback | --- diff --git a/docs/choose-bindings.md b/docs/choose-bindings.md index c44820671..edb5f2526 100644 --- a/docs/choose-bindings.md +++ b/docs/choose-bindings.md @@ -6,8 +6,8 @@ Pick the right one for your platform and use case: | | React Native | Flutter | Kotlin | Swift | Python | Rust | CLI | C++ | |---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| **Platforms** | iOS, Android | iOS, Android, macOS | Android, iOS (KMP) | iOS, macOS | Arm Linux, macOS | All | macOS, Linux | All | -| **Install** | npm | build | build | build | build | cargo | brew / source | header | +| **Platforms** | iOS, Android | iOS, Android, macOS | Android, iOS (KMP) | iOS, macOS | Arm Linux, macOS | Arm Linux, macOS | macOS, Arm Linux | Arm Linux, macOS, iOS, Android | +| **Install** | build / source | build | build | build | pip | build / source | brew / source | header | | **LLM** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | **Streaming** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | **Vision** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | @@ -35,12 +35,12 @@ Pick the right one for your platform and use case: ## Binding Documentation -- **[React Native](https://github.com/cactus-compute/cactus-react-native)** -- npm package with React hooks (`useCactusLM`) +- **[React Native](/bindings/react-native/)** -- Native bridge modules over the C API for iOS and Android - **[Python](/python/)** -- Module-level FFI bindings, mirrors the C API - **[Swift](/bindings/swift/)** -- XCFramework for iOS/macOS with NPU support - **[Kotlin](/bindings/kotlin/)** -- JNI bindings + Kotlin Multiplatform support -- **[Flutter](/flutter/)** -- Dart FFI bindings for Android, iOS, and macOS -- **[Rust](/rust/)** -- Auto-generated FFI bindings via bindgen +- **[Flutter](/bindings/flutter/)** -- Dart FFI bindings for Android, iOS, and macOS +- **[Rust](/bindings/rust/)** -- Raw `extern "C"` FFI declarations - **[C++ / Engine API](/docs/cactus_engine.md)** -- Direct C FFI for maximum control ## Getting Started diff --git a/docs/choose-sdk.md b/docs/choose-sdk.md deleted file mode 100644 index df5be4205..000000000 --- a/docs/choose-sdk.md +++ /dev/null @@ -1,49 +0,0 @@ -# Choose Your SDK - -Not sure which SDK to use? - -Pick the right one for your platform and use case: - -| | React Native | Flutter | Kotlin | Swift | Python | Rust | CLI | C++ | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| **Platforms** | iOS, Android | iOS, Android, macOS | Android, iOS (KMP) | iOS, macOS | Arm Linux, macOS | All | macOS, Linux | All | -| **Install** | npm | build | build | build | build | cargo | brew / source | header | -| **LLM** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Streaming** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Vision** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Audio (Gemma4)** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Transcription** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Diarization** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Function Calling** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **RAG / Embeddings** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | -| **Cloud Fallback** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | - -## Quick Recommendations - -!!! tip "Building a mobile app?" - - **React Native** -- already have a React Native project? Just `npm install` and go - - **Flutter** -- cross-platform mobile + Mac with full native bindings - - **Kotlin** -- native Android apps or Kotlin Multiplatform - - **Swift** -- native iOS/macOS apps with Apple NPU acceleration - -!!! tip "Server-side or scripting?" - - **Python** -- server-side inference, batch processing, or rapid prototyping - - **CLI** -- quick model testing and interactive sessions without writing code - -!!! tip "Embedding in a native app?" - - **C++** -- game engines, native desktop apps, or any C/C++ project - - **Rust** -- systems-level integration with safe FFI bindings - -## SDK Documentation - -- **[React Native](https://github.com/cactus-compute/cactus-react-native)** -- npm package with React hooks (`useCactusLM`) -- **[Python](/python/)** -- Module-level FFI bindings, mirrors the C API -- **[Swift](/apple/)** -- XCFramework for iOS/macOS with NPU support -- **[Kotlin / Android](/android/)** -- JNI bindings + Kotlin Multiplatform support -- **[Flutter](/flutter/)** -- Dart FFI bindings for Android, iOS, and macOS -- **[Rust](/rust/)** -- Auto-generated FFI bindings via bindgen -- **[C++ / Engine API](/docs/cactus_engine.md)** -- Direct C FFI for maximum control - -## Getting Started - -Once you've picked your SDK, head to the **[Quickstart](quickstart.md)** to install and run your first completion. diff --git a/docs/finetuning.md b/docs/finetuning.md index c15db43e5..25ad62ee4 100644 --- a/docs/finetuning.md +++ b/docs/finetuning.md @@ -9,7 +9,7 @@ keywords: ["fine-tuning", "LoRA", "Unsloth", "mobile deployment", "on-device AI" - Cactus is an inference engine for mobile devices, macs and ARM chips like Raspberry Pi. - At INT8, Cactus runs `Qwen3-0.6B` and `LFM2-1.2B` at `60-70 toks/sec` on iPhone 17 Pro, `13-18 toks/sec` on budget Pixel 6a. - INT4 quantization provides ~50% memory reduction with minimal quality loss. -- Task-Specific INT8 tunes of `Gemma3-270m` hit `150 toks/sec` on iPhone 17 Pro and `23 toks/sec` on Raspberry Pi. +- Task-Specific INT8 tunes of `Gemma3-270m` hit `150 toks/sec` on iPhone 17 Pro and `23 toks/sec` on Raspberry Pi 5. ## Quick Start @@ -67,12 +67,17 @@ cactus convert Qwen/Qwen3-0.6B ./my-qwen3-0.6b --lora username/my-lora-adapter ### 4. Run -Build the native library, then test your model on Mac: +Build the native library, transpile the weights into a runtime bundle, then run: ```bash cactus build +cactus transpile Qwen/Qwen3-0.6B \ + --weights-dir ./my-qwen3-0.6b --artifact-dir ./my-qwen3-0.6b cactus run ./my-qwen3-0.6b ``` + +`cactus run` accepts the bundle path directly, or auto-builds from a HF model id +when no bundle exists locally. Logo ### 5. Use in iOS/macOS App @@ -86,36 +91,37 @@ cactus build --apple Build complete! Total time: 58 seconds Static libraries: - Device: /Users/henry/Desktop/cactus/apple/libcactus-device.a - Simulator: /Users/henry/Desktop/cactus/apple/libcactus-simulator.a + Device: /apple/libcactus_engine-device.a + Simulator: /apple/libcactus_engine-simulator.a XCFrameworks: - iOS: /Users/henry/Desktop/cactus/apple/cactus-ios.xcframework - macOS: /Users/henry/Desktop/cactus/apple/cactus-macos.xcframework -Apple build complete! -(venv) henry@Henrys-MacBook-Air cactus % + iOS: /apple/cactus-ios.xcframework + macOS: /apple/cactus-macos.xcframework +Building Cactus for Apple platforms complete! ``` Link `cactus-ios.xcframework` to your Xcode project, then: ```swift import Foundation -import cactus +import cactus // module map → cactus_engine.h -// Load model from app bundle let modelPath = Bundle.main.path(forResource: "my-model", ofType: nil)! -let model = try cactusInit(modelPath, nil, false) +let model = cactus_init(modelPath, nil, false) -// Run completion let messages = "[{\"role\":\"user\",\"content\":\"Hello!\"}]" -let resultJson = try cactusComplete(model, messages, nil, nil, nil) +var buf = [Int8](repeating: 0, count: 65536) +buf.withUnsafeMutableBufferPointer { ptr in + _ = cactus_complete(model, messages, ptr.baseAddress, ptr.count, + nil, nil, nil, nil, nil, 0) +} +let resultJson = String(cString: buf) if let data = resultJson.data(using: .utf8), let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], let response = obj["response"] as? String { print(response) } -// Cleanup -cactusDestroy(model) +cactus_destroy(model) ``` Logo @@ -124,7 +130,7 @@ but to see performance on any device while testing, run cactus tests by plugging any iphone to your Mac then running: ```bash -cactus test --model --ios +cactus test --model --transcription-model --ios ``` Cactus demo apps will eventually expand to using your custom fine-tunes. @@ -141,13 +147,12 @@ cactus build --android ``` ```bash Build complete! -Shared library location: /Users/henry/Desktop/cactus/android/libcactus.so -Static library location: /Users/henry/Desktop/cactus/android/libcactus.a -Android build complete! -(venv) henry@Henrys-MacBook-Air cactus % +Shared library location: /android/libcactus_engine.so +Static library location: /android/libcactus_engine.a +Building Cactus for Android complete! ``` -Copy `libcactus.so` to `app/src/main/jniLibs/arm64-v8a/`, then: +Copy `libcactus_engine.so` to `app/src/main/jniLibs/arm64-v8a/`, then: ```kotlin import com.cactus.* @@ -155,8 +160,14 @@ import org.json.JSONObject val model = cactusInit("/data/local/tmp/my-model", null, false) -// Run completion -val resultJson = cactusComplete(model, """[{"role":"user","content":"Hello!"}]""", null, null, null) +val resultJson = cactusComplete( + model, + """[{"role":"user","content":"Hello!"}]""", + null, // optionsJson + null, // toolsJson + null, // callback + null, // pcmData +) val response = JSONObject(resultJson).getString("response") println(response) cactusDestroy(model) @@ -167,7 +178,7 @@ but to see performance on any device while testing, run cactus tests by plugging any android phone to your Mac then running: ```bash -cactus test --model --android +cactus test --model --transcription-model --android ``` Cactus demo apps will eventually expand to using your custom fine-tunes. diff --git a/docs/quickstart.md b/docs/quickstart.md index 6c7a0f43f..8d7bfa9ed 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -6,9 +6,11 @@ Install Cactus and run your first on-device AI completion. === "React Native" - ```bash - npm install cactus-react-native react-native-nitro-modules - ``` + --8<-- "react-native/README.md:install" + + ### Platform Integration + + --8<-- "react-native/README.md:integration" === "Flutter" @@ -20,19 +22,19 @@ Install Cactus and run your first on-device AI completion. === "Kotlin" - --8<-- "android/README.md:install" + --8<-- "kotlin/README.md:install" ### Platform Integration - --8<-- "android/README.md:integration" + --8<-- "kotlin/README.md:integration" === "Swift" - --8<-- "apple/README.md:install" + --8<-- "swift/README.md:install" ### Platform Integration - --8<-- "apple/README.md:integration" + --8<-- "swift/README.md:integration" === "Python" @@ -53,14 +55,15 @@ Install Cactus and run your first on-device AI completion. **From Source (macOS):** ```bash - git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup + brew install cmake + git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup && cactus build --python ``` **From Source (Linux):** ```bash sudo apt-get install python3 python3-venv python3-pip cmake build-essential libcurl4-openssl-dev - git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup + git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup && cactus build --python ``` === "C++" @@ -68,7 +71,7 @@ Install Cactus and run your first on-device AI completion. Include the Cactus header in your project: ```cpp - #include + #include ``` See the [Cactus repository](https://github.com/cactus-compute/cactus) for CMake build instructions. @@ -79,36 +82,7 @@ Install Cactus and run your first on-device AI completion. === "React Native" - ```tsx - import { useCactusLM } from 'cactus-react-native'; - - const App = () => { - const cactusLM = useCactusLM(); - - useEffect(() => { - if (!cactusLM.isDownloaded) { - cactusLM.download(); - } - }, []); - - const handleGenerate = () => { - cactusLM.complete({ - messages: [{ role: 'user', content: 'What is the capital of France?' }], - }); - }; - - if (cactusLM.isDownloading) { - return Downloading: {Math.round(cactusLM.downloadProgress * 100)}%; - } - - return ( - <> -